From 386d2faff172b94d320040c3a206ed072a3f436d Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 10 Jul 2025 14:47:44 +0000 Subject: [PATCH 001/336] Preparing development version 2.6.4-SNAPSHOT Signed-off-by: Duo Zhang --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6ee14697144..825e76190640 100644 --- a/pom.xml +++ b/pom.xml @@ -523,7 +523,7 @@ - 2.6.3 + 2.6.4-SNAPSHOT false From 927f077abaa7c03b46d9c3f0ca5420a1e9b27da7 Mon Sep 17 00:00:00 2001 From: sanjeet006py <36011005+sanjeet006py@users.noreply.github.com> Date: Sat, 19 Jul 2025 00:48:59 +0530 Subject: [PATCH 002/336] HBASE-29398: Server side scan metrics for bytes read from FS vs Block cache vs memstore (#7162) (#7136) Signed-off-by: Viraj Jasani Signed-off-by: Hari Krishna Dara --- .../client/metrics/ServerSideScanMetrics.java | 19 + .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 1 + .../hbase/io/hfile/CompoundBloomFilter.java | 18 +- .../hbase/io/hfile/FixedFileTrailer.java | 10 +- .../hadoop/hbase/io/hfile/HFileBlock.java | 24 +- .../hbase/io/hfile/HFileBlockIndex.java | 7 +- .../hbase/io/hfile/HFileReaderImpl.java | 130 +-- .../hadoop/hbase/io/hfile/LruBlockCache.java | 5 +- .../hbase/io/hfile/NoOpIndexBlockEncoder.java | 4 +- .../ThreadLocalServerSideScanMetrics.java | 160 ++++ .../hbase/regionserver/RegionScannerImpl.java | 27 +- .../hbase/regionserver/SegmentScanner.java | 13 + .../hbase/regionserver/StoreFileReader.java | 3 +- .../hbase/regionserver/StoreFileWriter.java | 4 +- .../hbase/regionserver/StoreScanner.java | 38 + .../handler/ParallelSeekHandler.java | 36 + .../hbase/io/hfile/TestBytesReadFromFs.java | 412 ++++++++ .../hadoop/hbase/io/hfile/TestHFile.java | 85 ++ .../TestBytesReadServerSideScanMetrics.java | 894 ++++++++++++++++++ .../regionserver/TestDefaultMemStore.java | 44 + 20 files changed, 1866 insertions(+), 68 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBytesReadServerSideScanMetrics.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java index 916451df75da..6b3a4f5675ad 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java @@ -52,6 +52,10 @@ public void moveToNextRegion() { currentRegionScanMetricsData.createCounter(COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME); currentRegionScanMetricsData.createCounter(BLOCK_BYTES_SCANNED_KEY_METRIC_NAME); currentRegionScanMetricsData.createCounter(FS_READ_TIME_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_FS_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_MEMSTORE_METRIC_NAME); + currentRegionScanMetricsData.createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); } /** @@ -68,6 +72,11 @@ protected AtomicLong createCounter(String counterName) { public static final String BLOCK_BYTES_SCANNED_KEY_METRIC_NAME = "BLOCK_BYTES_SCANNED"; public static final String FS_READ_TIME_METRIC_NAME = "FS_READ_TIME"; + public static final String BYTES_READ_FROM_FS_METRIC_NAME = "BYTES_READ_FROM_FS"; + public static final String BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME = + "BYTES_READ_FROM_BLOCK_CACHE"; + public static final String BYTES_READ_FROM_MEMSTORE_METRIC_NAME = "BYTES_READ_FROM_MEMSTORE"; + public static final String BLOCK_READ_OPS_COUNT_METRIC_NAME = "BLOCK_READ_OPS_COUNT"; /** * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 @@ -102,6 +111,16 @@ protected AtomicLong createCounter(String counterName) { public final AtomicLong fsReadTime = createCounter(FS_READ_TIME_METRIC_NAME); + public final AtomicLong bytesReadFromFs = createCounter(BYTES_READ_FROM_FS_METRIC_NAME); + + public final AtomicLong bytesReadFromBlockCache = + createCounter(BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME); + + public final AtomicLong bytesReadFromMemstore = + createCounter(BYTES_READ_FROM_MEMSTORE_METRIC_NAME); + + public final AtomicLong blockReadOpsCount = createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); + /** * Sets counter with counterName to passed in value, does nothing if counter does not exist. If * region level scan metrics are enabled then sets the value of counter for the current region diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index b130d06ca43e..c963fc2617fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -288,6 +288,7 @@ public static HFileBlock getBlockForCaching(CacheConfig cacheConf, HFileBlock bl .withFillHeader(FILL_HEADER).withOffset(block.getOffset()).withNextBlockOnDiskSize(-1) .withOnDiskDataSizeWithHeader(block.getOnDiskDataSizeWithHeader() + numBytes) .withHFileContext(cloneContext(block.getHFileContext())) + .withNextBlockOnDiskSize(block.getNextBlockOnDiskSize()) .withByteBuffAllocator(cacheConf.getByteBuffAllocator()).withShared(!buff.hasArray()).build(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java index 95bc1c7b83d5..b1cb519fb585 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java @@ -20,6 +20,7 @@ import java.io.DataInput; import java.io.IOException; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.BloomFilter; @@ -120,7 +121,8 @@ private boolean containsInternal(byte[] key, int keyOffset, int keyLength, ByteB return result; } - private HFileBlock getBloomBlock(int block) { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public HFileBlock getBloomBlock(int block) { HFileBlock bloomBlock; try { // We cache the block and use a positional read. @@ -218,4 +220,18 @@ public String toString() { return sb.toString(); } + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public HFileBlockIndex.BlockIndexReader getBloomIndex() { + return index; + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public int getHashCount() { + return hashCount; + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public Hash getHash() { + return hash; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java index f2071d21abfd..fe0b3124cb54 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java @@ -27,10 +27,12 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.InnerStoreCellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MetaCellComparator; import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -410,6 +412,11 @@ public static FixedFileTrailer readFromStream(FSDataInputStream istream, long fi FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion); fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(), buf.arrayOffset() + bufferSize - trailerSize, trailerSize))); + boolean isScanMetricsEnabled = ThreadLocalServerSideScanMetrics.isScanMetricsEnabled(); + if (isScanMetricsEnabled) { + ThreadLocalServerSideScanMetrics.addBytesReadFromFs(trailerSize); + ThreadLocalServerSideScanMetrics.addBlockReadOpsCount(1); + } return fft; } @@ -648,7 +655,8 @@ static CellComparator createComparator(String comparatorClassName) throws IOExce } } - CellComparator createComparator() throws IOException { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public CellComparator createComparator() throws IOException { expectAtLeastMajorVersion(2); return createComparator(comparatorClassName); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index 036270824f71..122380c56496 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.ByteArrayOutputStream; @@ -56,6 +57,7 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; import org.apache.hadoop.hbase.io.hfile.trace.HFileContextAttributesBuilderConsumer; import org.apache.hadoop.hbase.io.util.BlockIOUtils; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.MultiByteBuff; import org.apache.hadoop.hbase.nio.SingleByteBuff; @@ -405,7 +407,8 @@ private static int getOnDiskSizeWithHeader(final ByteBuff headerBuf, boolean che * present) read by peeking into the next block's header; use as a hint when doing a read * of the next block when scanning or running over a file. */ - int getNextBlockOnDiskSize() { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public int getNextBlockOnDiskSize() { return nextBlockOnDiskSize; } @@ -626,7 +629,8 @@ public String toString() { * Retrieves the decompressed/decrypted view of this block. An encoded block remains in its * encoded structure. Internal structures are shared between instances where applicable. */ - HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException { if (!fileContext.isCompressedOrEncrypted()) { // TODO: cannot use our own fileContext here because HFileBlock(ByteBuffer, boolean), // which is used for block serialization to L2 cache, does not preserve encoding and @@ -1241,7 +1245,8 @@ interface BlockWritable { * Iterator for reading {@link HFileBlock}s in load-on-open-section, such as root data index * block, meta index block, file info block etc. */ - interface BlockIterator { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public interface BlockIterator { /** * Get the next block, or null if there are no more blocks to iterate. */ @@ -1265,7 +1270,8 @@ interface BlockIterator { } /** An HFile block reader with iteration ability. */ - interface FSReader { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public interface FSReader { /** * Reads the block at the given offset in the file with the given on-disk size and uncompressed * size. @@ -1738,6 +1744,7 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, // checksums. Can change with circumstances. The below flag is whether the // file has support for checksums (version 2+). boolean checksumSupport = this.fileContext.isUseHBaseChecksum(); + boolean isScanMetricsEnabled = ThreadLocalServerSideScanMetrics.isScanMetricsEnabled(); long startTime = EnvironmentEdgeManager.currentTime(); if (onDiskSizeWithHeader == -1) { // The caller does not know the block size. Need to get it from the header. If header was @@ -1754,6 +1761,9 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, headerBuf = HEAP.allocate(hdrSize); readAtOffset(is, headerBuf, hdrSize, false, offset, pread); headerBuf.rewind(); + if (isScanMetricsEnabled) { + ThreadLocalServerSideScanMetrics.addBytesReadFromFs(hdrSize); + } } onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf, checksumSupport); } @@ -1801,6 +1811,12 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, boolean readNextHeader = readAtOffset(is, onDiskBlock, onDiskSizeWithHeader - preReadHeaderSize, true, offset + preReadHeaderSize, pread); onDiskBlock.rewind(); // in case of moving position when copying a cached header + if (isScanMetricsEnabled) { + long bytesRead = + (onDiskSizeWithHeader - preReadHeaderSize) + (readNextHeader ? hdrSize : 0); + ThreadLocalServerSideScanMetrics.addBytesReadFromFs(bytesRead); + ThreadLocalServerSideScanMetrics.addBlockReadOpsCount(1); + } // the call to validateChecksum for this block excludes the next block header over-read, so // no reason to delay extracting this value. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index 8989ef2b2814..2858800e9322 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KeyOnlyKeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -561,7 +562,8 @@ public String toString() { * array of offsets to the entries within the block. This allows us to do binary search for the * entry corresponding to the given key without having to deserialize the block. */ - static abstract class BlockIndexReader implements HeapSize { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public static abstract class BlockIndexReader implements HeapSize { protected long[] blockOffsets; protected int[] blockDataSizes; @@ -808,7 +810,8 @@ static int binarySearchNonRootIndex(Cell key, ByteBuff nonRootIndex, * @return the index position where the given key was found, otherwise return -1 in the case the * given key is before the first key. */ - static int locateNonRootIndexEntry(ByteBuff nonRootBlock, Cell key, CellComparator comparator) { + public static int locateNonRootIndexEntry(ByteBuff nonRootBlock, Cell key, + CellComparator comparator) { int entryIndex = binarySearchNonRootIndex(key, nonRootBlock, comparator); if (entryIndex != -1) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index db2383db399d..e1e9eaf8a53a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -45,6 +46,7 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -1105,71 +1107,91 @@ public void setConf(Configuration conf) { * Retrieve block from cache. Validates the retrieved block's type vs {@code expectedBlockType} * and its encoding vs. {@code expectedDataBlockEncoding}. Unpacks the block as necessary. */ - private HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock, + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boolean useLock, boolean updateCacheMetrics, BlockType expectedBlockType, DataBlockEncoding expectedDataBlockEncoding) throws IOException { // Check cache for block. If found return. BlockCache cache = cacheConf.getBlockCache().orElse(null); + long cachedBlockBytesRead = 0; if (cache != null) { - HFileBlock cachedBlock = - (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); - if (cachedBlock != null) { - if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { - HFileBlock compressedBlock = cachedBlock; - cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader); - // In case of compressed block after unpacking we can release the compressed block - if (compressedBlock != cachedBlock) { - compressedBlock.release(); + HFileBlock cachedBlock = null; + boolean isScanMetricsEnabled = ThreadLocalServerSideScanMetrics.isScanMetricsEnabled(); + try { + cachedBlock = + (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); + if (cachedBlock != null) { + if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { + HFileBlock compressedBlock = cachedBlock; + cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader); + // In case of compressed block after unpacking we can release the compressed block + if (compressedBlock != cachedBlock) { + compressedBlock.release(); + } + } + try { + validateBlockType(cachedBlock, expectedBlockType); + } catch (IOException e) { + returnAndEvictBlock(cache, cacheKey, cachedBlock); + cachedBlock = null; + throw e; } - } - try { - validateBlockType(cachedBlock, expectedBlockType); - } catch (IOException e) { - returnAndEvictBlock(cache, cacheKey, cachedBlock); - throw e; - } - if (expectedDataBlockEncoding == null) { - return cachedBlock; - } - DataBlockEncoding actualDataBlockEncoding = cachedBlock.getDataBlockEncoding(); - // Block types other than data blocks always have - // DataBlockEncoding.NONE. To avoid false negative cache misses, only - // perform this check if cached block is a data block. - if ( - cachedBlock.getBlockType().isData() - && !actualDataBlockEncoding.equals(expectedDataBlockEncoding) - ) { - // This mismatch may happen if a Scanner, which is used for say a - // compaction, tries to read an encoded block from the block cache. - // The reverse might happen when an EncodedScanner tries to read - // un-encoded blocks which were cached earlier. - // - // Because returning a data block with an implicit BlockType mismatch - // will cause the requesting scanner to throw a disk read should be - // forced here. This will potentially cause a significant number of - // cache misses, so update so we should keep track of this as it might - // justify the work on a CompoundScanner. + if (expectedDataBlockEncoding == null) { + return cachedBlock; + } + DataBlockEncoding actualDataBlockEncoding = cachedBlock.getDataBlockEncoding(); + // Block types other than data blocks always have + // DataBlockEncoding.NONE. To avoid false negative cache misses, only + // perform this check if cached block is a data block. if ( - !expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) - && !actualDataBlockEncoding.equals(DataBlockEncoding.NONE) + cachedBlock.getBlockType().isData() + && !actualDataBlockEncoding.equals(expectedDataBlockEncoding) ) { - // If the block is encoded but the encoding does not match the - // expected encoding it is likely the encoding was changed but the - // block was not yet evicted. Evictions on file close happen async - // so blocks with the old encoding still linger in cache for some - // period of time. This event should be rare as it only happens on - // schema definition change. - LOG.info( - "Evicting cached block with key {} because data block encoding mismatch; " - + "expected {}, actual {}, path={}", - cacheKey, actualDataBlockEncoding, expectedDataBlockEncoding, path); - // This is an error scenario. so here we need to release the block. - returnAndEvictBlock(cache, cacheKey, cachedBlock); + // This mismatch may happen if a Scanner, which is used for say a + // compaction, tries to read an encoded block from the block cache. + // The reverse might happen when an EncodedScanner tries to read + // un-encoded blocks which were cached earlier. + // + // Because returning a data block with an implicit BlockType mismatch + // will cause the requesting scanner to throw a disk read should be + // forced here. This will potentially cause a significant number of + // cache misses, so update so we should keep track of this as it might + // justify the work on a CompoundScanner. + if ( + !expectedDataBlockEncoding.equals(DataBlockEncoding.NONE) + && !actualDataBlockEncoding.equals(DataBlockEncoding.NONE) + ) { + // If the block is encoded but the encoding does not match the + // expected encoding it is likely the encoding was changed but the + // block was not yet evicted. Evictions on file close happen async + // so blocks with the old encoding still linger in cache for some + // period of time. This event should be rare as it only happens on + // schema definition change. + LOG.info( + "Evicting cached block with key {} because data block encoding mismatch; " + + "expected {}, actual {}, path={}", + cacheKey, actualDataBlockEncoding, expectedDataBlockEncoding, path); + // This is an error scenario. so here we need to release the block. + returnAndEvictBlock(cache, cacheKey, cachedBlock); + } + cachedBlock = null; + return null; } - return null; + return cachedBlock; + } + } finally { + // Count bytes read as cached block is being returned + if (isScanMetricsEnabled && cachedBlock != null) { + cachedBlockBytesRead = cachedBlock.getOnDiskSizeWithHeader(); + // Account for the header size of the next block if it exists + if (cachedBlock.getNextBlockOnDiskSize() > 0) { + cachedBlockBytesRead += cachedBlock.headerSize(); + } + } + if (cachedBlockBytesRead > 0) { + ThreadLocalServerSideScanMetrics.addBytesReadFromBlockCache(cachedBlockBytesRead); } - return cachedBlock; } } return null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index b61084f78836..efc533cb1497 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -36,6 +36,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.ClassSize; @@ -1151,8 +1152,8 @@ public void remove() { } // Simple calculators of sizes given factors and maxSize - - long acceptableSize() { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public long acceptableSize() { return (long) Math.floor(this.maxSize * this.acceptableFactor); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpIndexBlockEncoder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpIndexBlockEncoder.java index 4162fca6afe5..fd4bcc12c503 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpIndexBlockEncoder.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpIndexBlockEncoder.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding; @@ -127,7 +128,8 @@ public String toString() { return getClass().getSimpleName(); } - protected static class NoOpEncodedSeeker implements EncodedSeeker { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public static class NoOpEncodedSeeker implements EncodedSeeker { protected long[] blockOffsets; protected int[] blockDataSizes; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java new file mode 100644 index 000000000000..8c9ec24e8662 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.ScannerContext; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Thread-local storage for server-side scan metrics that captures performance data separately for + * each scan thread. This class works in conjunction with {@link ServerSideScanMetrics} to provide + * comprehensive scan performance monitoring. + *

Purpose and Design

{@link ServerSideScanMetrics} captures scan metrics on the server + * side and passes them back to the client in protocol buffer responses. However, the + * {@link ServerSideScanMetrics} instance is not readily available at deeper layers in HBase where + * HFiles are read and individual HFile blocks are accessed. To avoid the complexity of passing + * {@link ServerSideScanMetrics} instances through method calls across multiple layers, this class + * provides thread-local storage for metrics collection. + *

Thread Safety and HBase Architecture

This class leverages a critical aspect of HBase + * server design: on the server side, the thread that opens a {@link RegionScanner} and calls + * {@link RegionScanner#nextRaw(java.util.List, ScannerContext)} is the same thread that reads HFile + * blocks. This design allows thread-local storage to effectively capture metrics without + * cross-thread synchronization. + *

Special Handling for Parallel Operations

The only deviation from the single-thread model + * occurs when {@link org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler} is used for + * parallel store file seeking. In this case, special handling ensures that metrics are captured + * correctly across multiple threads. The + * {@link org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler} captures metrics from + * worker threads and aggregates them back to the main scan thread. Please refer to the javadoc of + * {@link org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler} for detailed information + * about this parallel processing mechanism. + *

Usage Pattern

+ *
    + *
  1. Enable metrics collection: {@link #setScanMetricsEnabled(boolean)}
    2. + *
  2. Reset counters at scan start: {@link #reset()}
    3. + *
  3. Increment counters during I/O operations using the various {@code add*} methods
    4. + *
  4. Populate the main metrics object: + * {@link #populateServerSideScanMetrics(ServerSideScanMetrics)}
    5. + *
+ *

Thread Safety

This class is thread-safe. Each thread maintains its own set of counters + * through {@link ThreadLocal} storage, ensuring that metrics from different scan operations do not + * interfere with each other. + * @see ServerSideScanMetrics + * @see RegionScanner + * @see org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler + */ +@InterfaceAudience.Private +public final class ThreadLocalServerSideScanMetrics { + private ThreadLocalServerSideScanMetrics() { + } + + private static final ThreadLocal IS_SCAN_METRICS_ENABLED = + ThreadLocal.withInitial(() -> false); + + private static final ThreadLocal BYTES_READ_FROM_FS = + ThreadLocal.withInitial(() -> new AtomicLong(0)); + + private static final ThreadLocal BYTES_READ_FROM_BLOCK_CACHE = + ThreadLocal.withInitial(() -> new AtomicLong(0)); + + private static final ThreadLocal BYTES_READ_FROM_MEMSTORE = + ThreadLocal.withInitial(() -> new AtomicLong(0)); + + private static final ThreadLocal BLOCK_READ_OPS_COUNT = + ThreadLocal.withInitial(() -> new AtomicLong(0)); + + public static void setScanMetricsEnabled(boolean enable) { + IS_SCAN_METRICS_ENABLED.set(enable); + } + + public static long addBytesReadFromFs(long bytes) { + return BYTES_READ_FROM_FS.get().addAndGet(bytes); + } + + public static long addBytesReadFromBlockCache(long bytes) { + return BYTES_READ_FROM_BLOCK_CACHE.get().addAndGet(bytes); + } + + public static long addBytesReadFromMemstore(long bytes) { + return BYTES_READ_FROM_MEMSTORE.get().addAndGet(bytes); + } + + public static long addBlockReadOpsCount(long count) { + return BLOCK_READ_OPS_COUNT.get().addAndGet(count); + } + + public static boolean isScanMetricsEnabled() { + return IS_SCAN_METRICS_ENABLED.get(); + } + + public static AtomicLong getBytesReadFromFsCounter() { + return BYTES_READ_FROM_FS.get(); + } + + public static AtomicLong getBytesReadFromBlockCacheCounter() { + return BYTES_READ_FROM_BLOCK_CACHE.get(); + } + + public static AtomicLong getBytesReadFromMemstoreCounter() { + return BYTES_READ_FROM_MEMSTORE.get(); + } + + public static AtomicLong getBlockReadOpsCountCounter() { + return BLOCK_READ_OPS_COUNT.get(); + } + + public static long getBytesReadFromFsAndReset() { + return getBytesReadFromFsCounter().getAndSet(0); + } + + public static long getBytesReadFromBlockCacheAndReset() { + return getBytesReadFromBlockCacheCounter().getAndSet(0); + } + + public static long getBytesReadFromMemstoreAndReset() { + return getBytesReadFromMemstoreCounter().getAndSet(0); + } + + public static long getBlockReadOpsCountAndReset() { + return getBlockReadOpsCountCounter().getAndSet(0); + } + + public static void reset() { + getBytesReadFromFsAndReset(); + getBytesReadFromBlockCacheAndReset(); + getBytesReadFromMemstoreAndReset(); + getBlockReadOpsCountAndReset(); + } + + public static void populateServerSideScanMetrics(ServerSideScanMetrics metrics) { + if (metrics == null) { + return; + } + metrics.addToCounter(ServerSideScanMetrics.BYTES_READ_FROM_FS_METRIC_NAME, + getBytesReadFromFsCounter().get()); + metrics.addToCounter(ServerSideScanMetrics.BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME, + getBytesReadFromBlockCacheCounter().get()); + metrics.addToCounter(ServerSideScanMetrics.BYTES_READ_FROM_MEMSTORE_METRIC_NAME, + getBytesReadFromMemstoreCounter().get()); + metrics.addToCounter(ServerSideScanMetrics.BLOCK_READ_OPS_COUNT_METRIC_NAME, + getBlockReadOpsCountCounter().get()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java index 67cc9b7eba48..153900544d19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcCallback; import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState; @@ -94,6 +95,8 @@ public class RegionScannerImpl implements RegionScanner, Shipper, RpcCallback { private RegionServerServices rsServices; + private ServerSideScanMetrics scannerInitMetrics = null; + @Override public RegionInfo getRegionInfo() { return region.getRegionInfo(); @@ -144,7 +147,16 @@ private static boolean hasNonce(HRegion region, long nonce) { } finally { region.smallestReadPointCalcLock.unlock(ReadPointCalculationLock.LockType.RECORDING_LOCK); } + boolean isScanMetricsEnabled = scan.isScanMetricsEnabled(); + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(isScanMetricsEnabled); + if (isScanMetricsEnabled) { + this.scannerInitMetrics = new ServerSideScanMetrics(); + ThreadLocalServerSideScanMetrics.reset(); + } initializeScanners(scan, additionalScanners); + if (isScanMetricsEnabled) { + ThreadLocalServerSideScanMetrics.populateServerSideScanMetrics(scannerInitMetrics); + } } public ScannerContext getContext() { @@ -277,6 +289,16 @@ public boolean nextRaw(List outResults, ScannerContext scannerContext) thr throw new UnknownScannerException("Scanner was closed"); } boolean moreValues = false; + boolean isScanMetricsEnabled = scannerContext.isTrackingMetrics(); + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(isScanMetricsEnabled); + if (isScanMetricsEnabled) { + ThreadLocalServerSideScanMetrics.reset(); + ServerSideScanMetrics scanMetrics = scannerContext.getMetrics(); + if (scannerInitMetrics != null) { + scannerInitMetrics.getMetricsMap().forEach(scanMetrics::addToCounter); + scannerInitMetrics = null; + } + } if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called // to handle scan or get operation. @@ -286,7 +308,10 @@ public boolean nextRaw(List outResults, ScannerContext scannerContext) thr moreValues = nextInternal(tmpList, scannerContext); outResults.addAll(tmpList); } - + if (isScanMetricsEnabled) { + ServerSideScanMetrics scanMetrics = scannerContext.getMetrics(); + ThreadLocalServerSideScanMetrics.populateServerSideScanMetrics(scanMetrics); + } region.addReadRequestsCount(1); if (region.getMetrics() != null) { region.getMetrics().updateReadRequestCount(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java index 1d28c55570ed..27df80fa8ff8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.yetus.audience.InterfaceAudience; /** @@ -54,6 +55,7 @@ public class SegmentScanner implements KeyValueScanner { // flag to indicate if this scanner is closed protected boolean closed = false; + private boolean isScanMetricsEnabled = false; /** * Scanners are ordered from 0 (oldest) to newest in increasing order. @@ -66,6 +68,8 @@ protected SegmentScanner(Segment segment, long readPoint) { iter = segment.iterator(); // the initialization of the current is required for working with heap of SegmentScanners updateCurrent(); + // Enable scan metrics for tracking bytes read after initialization of current + this.isScanMetricsEnabled = ThreadLocalServerSideScanMetrics.isScanMetricsEnabled(); if (current == null) { // nothing to fetch from this scanner close(); @@ -335,10 +339,15 @@ private Segment getSegment() { */ protected void updateCurrent() { Cell next = null; + long totalBytesRead = 0; try { while (iter.hasNext()) { next = iter.next(); + if (isScanMetricsEnabled) { + // Batch collect bytes to reduce method call overhead + totalBytesRead += Segment.getCellLength(next); + } if (next.getSequenceId() <= this.readPoint) { current = next; return;// skip irrelevant versions @@ -352,6 +361,10 @@ protected void updateCurrent() { current = null; // nothing found } finally { + // Add accumulated bytes before returning + if (totalBytesRead > 0) { + ThreadLocalServerSideScanMetrics.addBytesReadFromMemstore(totalBytesRead); + } if (next != null) { // in all cases, remember the last KV we iterated to, needed for reseek() last = next; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java index e241bf0a5d34..fac344c3387b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java @@ -635,7 +635,8 @@ public boolean isBulkLoaded() { return this.bulkLoadResult; } - BloomFilter getGeneralBloomFilter() { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public BloomFilter getGeneralBloomFilter() { return generalBloomFilter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java index 61d5b91b35b6..a2a641df6d8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -284,7 +285,8 @@ public boolean hasGeneralBloom() { * For unit testing only. * @return the Bloom filter used by this writer. */ - BloomFilterWriter getGeneralBloomWriter() { + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + public BloomFilterWriter getGeneralBloomWriter() { return liveFileWriter.generalBloomFilterWriter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index cbc617d2a0bc..506af8c404b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -24,12 +24,14 @@ import java.util.NavigableSet; import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import java.util.function.IntConsumer; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; @@ -165,6 +167,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner protected final long readPt; private boolean topChanged = false; + // These are used to verify the state of the scanner during testing. + private static AtomicBoolean hasUpdatedReaders; + private static AtomicBoolean hasSwitchedToStreamRead; + /** An internal constructor. */ private StoreScanner(HStore store, Scan scan, ScanInfo scanInfo, int numColumns, long readPt, boolean cacheBlocks, ScanType scanType) { @@ -1016,6 +1022,9 @@ public void updateReaders(List sfs, List memStoreSc if (updateReaders) { closeLock.unlock(); } + if (hasUpdatedReaders != null) { + hasUpdatedReaders.set(true); + } } // Let the next() call handle re-creating and seeking } @@ -1166,6 +1175,9 @@ void trySwitchToStreamRead() { this.heap = newHeap; resetQueryMatcher(lastTop); scannersToClose.forEach(KeyValueScanner::close); + if (hasSwitchedToStreamRead != null) { + hasSwitchedToStreamRead.set(true); + } } protected final boolean checkFlushed() { @@ -1273,4 +1285,30 @@ public void shipped() throws IOException { trySwitchToStreamRead(); } } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + static final void instrument() { + hasUpdatedReaders = new AtomicBoolean(false); + hasSwitchedToStreamRead = new AtomicBoolean(false); + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + static final boolean hasUpdatedReaders() { + return hasUpdatedReaders.get(); + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + static final boolean hasSwitchedToStreamRead() { + return hasSwitchedToStreamRead.get(); + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + static final void resetHasUpdatedReaders() { + hasUpdatedReaders.set(false); + } + + @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) + static final void resetHasSwitchedToStreamRead() { + hasSwitchedToStreamRead.set(false); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java index 41fb3e7bf12b..9a9af221b49a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java @@ -19,9 +19,11 @@ import java.io.IOException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.EventType; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -39,6 +41,17 @@ public class ParallelSeekHandler extends EventHandler { private CountDownLatch latch; private Throwable err = null; + // Flag to enable/disable scan metrics collection and thread-local counters for capturing scan + // performance during parallel store file seeking. + // These aggregate metrics from worker threads back to the main scan thread. + private final boolean isScanMetricsEnabled; + // Thread-local counter for bytes read from FS. + private final AtomicLong bytesReadFromFs; + // Thread-local counter for bytes read from BlockCache. + private final AtomicLong bytesReadFromBlockCache; + // Thread-local counter for block read operations count. + private final AtomicLong blockReadOpsCount; + public ParallelSeekHandler(KeyValueScanner scanner, Cell keyValue, long readPoint, CountDownLatch latch) { super(null, EventType.RS_PARALLEL_SEEK); @@ -46,12 +59,35 @@ public ParallelSeekHandler(KeyValueScanner scanner, Cell keyValue, long readPoin this.keyValue = keyValue; this.readPoint = readPoint; this.latch = latch; + this.isScanMetricsEnabled = ThreadLocalServerSideScanMetrics.isScanMetricsEnabled(); + this.bytesReadFromFs = ThreadLocalServerSideScanMetrics.getBytesReadFromFsCounter(); + this.bytesReadFromBlockCache = + ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheCounter(); + this.blockReadOpsCount = ThreadLocalServerSideScanMetrics.getBlockReadOpsCountCounter(); } @Override public void process() { try { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(isScanMetricsEnabled); + if (isScanMetricsEnabled) { + ThreadLocalServerSideScanMetrics.reset(); + } scanner.seek(keyValue); + if (isScanMetricsEnabled) { + long metricValue = ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + if (metricValue > 0) { + bytesReadFromFs.addAndGet(metricValue); + } + metricValue = ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset(); + if (metricValue > 0) { + bytesReadFromBlockCache.addAndGet(metricValue); + } + metricValue = ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); + if (metricValue > 0) { + blockReadOpsCount.addAndGet(metricValue); + } + } } catch (IOException e) { LOG.error("", e); setErr(e); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java new file mode 100644 index 000000000000..c85a162ad96a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.StoreFileReader; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.BloomFilter; +import org.apache.hadoop.hbase.util.BloomFilterFactory; +import org.apache.hadoop.hbase.util.BloomFilterUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ IOTests.class, SmallTests.class }) +public class TestBytesReadFromFs { + private static final int NUM_KEYS = 100000; + private static final int BLOOM_BLOCK_SIZE = 512; + private static final int INDEX_CHUNK_SIZE = 512; + private static final int DATA_BLOCK_SIZE = 4096; + private static final int ROW_PREFIX_LENGTH_IN_BLOOM_FILTER = 42; + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBytesReadFromFs.class); + + @Rule + public TestName name = new TestName(); + + private static final Logger LOG = LoggerFactory.getLogger(TestBytesReadFromFs.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Random RNG = new Random(9713312); // Just a fixed seed. + + private Configuration conf; + private FileSystem fs; + private List keyValues = new ArrayList<>(); + private List keyList = new ArrayList<>(); + private Path path; + + @Before + public void setUp() throws IOException { + conf = TEST_UTIL.getConfiguration(); + conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, ROW_PREFIX_LENGTH_IN_BLOOM_FILTER); + fs = FileSystem.get(conf); + String hfileName = UUID.randomUUID().toString().replaceAll("-", ""); + path = new Path(TEST_UTIL.getDataTestDir(), hfileName); + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_CHUNK_SIZE); + } + + @Test + public void testBytesReadFromFsWithScanMetricsDisabled() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false); + writeData(path); + KeyValue keyValue = keyValues.get(0); + readDataAndIndexBlocks(path, keyValue, false); + } + + @Test + public void testBytesReadFromFsToReadDataUsingIndexBlocks() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + writeData(path); + KeyValue keyValue = keyValues.get(0); + readDataAndIndexBlocks(path, keyValue, true); + } + + @Test + public void testBytesReadFromFsToReadLoadOnOpenDataSection() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + writeData(path); + readLoadOnOpenDataSection(path, false); + } + + @Test + public void testBytesReadFromFsToReadBloomFilterIndexesAndBloomBlocks() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + BloomType[] bloomTypes = { BloomType.ROW, BloomType.ROWCOL, BloomType.ROWPREFIX_FIXED_LENGTH }; + for (BloomType bloomType : bloomTypes) { + LOG.info("Testing bloom type: {}", bloomType); + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); + keyList.clear(); + keyValues.clear(); + writeBloomFilters(path, bloomType, BLOOM_BLOCK_SIZE); + if (bloomType == BloomType.ROWCOL) { + KeyValue keyValue = keyValues.get(0); + readBloomFilters(path, bloomType, null, keyValue); + } else { + Assert.assertEquals(ROW_PREFIX_LENGTH_IN_BLOOM_FILTER, keyList.get(0).length); + byte[] key = keyList.get(0); + readBloomFilters(path, bloomType, key, null); + } + } + } + + private void writeData(Path path) throws IOException { + HFileContext context = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE) + .withIncludesTags(false).withDataBlockEncoding(DataBlockEncoding.NONE) + .withCompression(Compression.Algorithm.NONE).build(); + CacheConfig cacheConfig = new CacheConfig(conf); + HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig).withPath(fs, path) + .withFileContext(context).create(); + + byte[] cf = Bytes.toBytes("cf"); + byte[] cq = Bytes.toBytes("cq"); + + for (int i = 0; i < NUM_KEYS; i++) { + byte[] keyBytes = RandomKeyValueUtil.randomOrderedFixedLengthKey(RNG, i, 10); + // A random-length random value. + byte[] valueBytes = RandomKeyValueUtil.randomFixedLengthValue(RNG, 10); + KeyValue keyValue = + new KeyValue(keyBytes, cf, cq, EnvironmentEdgeManager.currentTime(), valueBytes); + writer.append(keyValue); + keyValues.add(keyValue); + } + + writer.close(); + } + + private void readDataAndIndexBlocks(Path path, KeyValue keyValue, boolean isScanMetricsEnabled) + throws IOException { + long fileSize = fs.getFileStatus(path).getLen(); + + ReaderContext readerContext = + new ReaderContextBuilder().withInputStreamWrapper(new FSDataInputStreamWrapper(fs, path)) + .withFilePath(path).withFileSystem(fs).withFileSize(fileSize).build(); + + // Read HFile trailer and create HFileContext + HFileInfo hfile = new HFileInfo(readerContext, conf); + FixedFileTrailer trailer = hfile.getTrailer(); + + // Read HFile info and load-on-open data section (we will read root again explicitly later) + CacheConfig cacheConfig = new CacheConfig(conf); + HFile.Reader reader = new HFilePreadReader(readerContext, hfile, cacheConfig, conf); + hfile.initMetaAndIndex(reader); + HFileContext meta = hfile.getHFileContext(); + + // Get access to the block reader + HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); + + // Create iterator for reading load-on-open data section + HFileBlock.BlockIterator blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), + fileSize - trailer.getTrailerSize()); + + // Indexes use NoOpEncodedSeeker + MyNoOpEncodedSeeker seeker = new MyNoOpEncodedSeeker(); + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); + + int bytesRead = 0; + int blockLevelsRead = 0; + + // Read the root index block + HFileBlock block = blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX); + bytesRead += block.getOnDiskSizeWithHeader(); + if (block.getNextBlockOnDiskSize() > 0) { + bytesRead += HFileBlock.headerSize(meta.isUseHBaseChecksum()); + } + blockLevelsRead++; + + // Comparator class name is stored in the trailer in version 3. + CellComparator comparator = trailer.createComparator(); + // Initialize the seeker + seeker.initRootIndex(block, trailer.getDataIndexCount(), comparator, + trailer.getNumDataIndexLevels()); + + int rootLevIndex = seeker.rootBlockContainingKey(keyValue); + long currentOffset = seeker.getBlockOffset(rootLevIndex); + int currentDataSize = seeker.getBlockDataSize(rootLevIndex); + + HFileBlock prevBlock = null; + do { + prevBlock = block; + block = blockReader.readBlockData(currentOffset, currentDataSize, true, true, true); + HFileBlock unpacked = block.unpack(meta, blockReader); + if (unpacked != block) { + block.release(); + block = unpacked; + } + bytesRead += block.getOnDiskSizeWithHeader(); + if (block.getNextBlockOnDiskSize() > 0) { + bytesRead += HFileBlock.headerSize(meta.isUseHBaseChecksum()); + } + if (!block.getBlockType().isData()) { + ByteBuff buffer = block.getBufferWithoutHeader(); + // Place the buffer at the correct position + HFileBlockIndex.BlockIndexReader.locateNonRootIndexEntry(buffer, keyValue, comparator); + currentOffset = buffer.getLong(); + currentDataSize = buffer.getInt(); + } + prevBlock.release(); + blockLevelsRead++; + } while (!block.getBlockType().isData()); + block.release(); + + reader.close(); + + Assert.assertEquals(isScanMetricsEnabled, + ThreadLocalServerSideScanMetrics.isScanMetricsEnabled()); + bytesRead = isScanMetricsEnabled ? bytesRead : 0; + Assert.assertEquals(bytesRead, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + Assert.assertEquals(blockLevelsRead, trailer.getNumDataIndexLevels() + 1); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); + // At every index level we read one index block and finally read data block + long blockReadOpsCount = isScanMetricsEnabled ? blockLevelsRead : 0; + Assert.assertEquals(blockReadOpsCount, + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset()); + } + + private void readLoadOnOpenDataSection(Path path, boolean hasBloomFilters) throws IOException { + long fileSize = fs.getFileStatus(path).getLen(); + + ReaderContext readerContext = + new ReaderContextBuilder().withInputStreamWrapper(new FSDataInputStreamWrapper(fs, path)) + .withFilePath(path).withFileSystem(fs).withFileSize(fileSize).build(); + + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); + // Read HFile trailer + HFileInfo hfile = new HFileInfo(readerContext, conf); + FixedFileTrailer trailer = hfile.getTrailer(); + Assert.assertEquals(trailer.getTrailerSize(), + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + Assert.assertEquals(1, ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset()); + + CacheConfig cacheConfig = new CacheConfig(conf); + HFile.Reader reader = new HFilePreadReader(readerContext, hfile, cacheConfig, conf); + HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); + + // Create iterator for reading root index block + HFileBlock.BlockIterator blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), + fileSize - trailer.getTrailerSize()); + boolean readNextHeader = false; + + // Read the root index block + readNextHeader = readEachBlockInLoadOnOpenDataSection( + blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), readNextHeader); + + // Read meta index block + readNextHeader = readEachBlockInLoadOnOpenDataSection( + blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), readNextHeader); + + // Read File info block + readNextHeader = readEachBlockInLoadOnOpenDataSection( + blockIter.nextBlockWithBlockType(BlockType.FILE_INFO), readNextHeader); + + // Read bloom filter indexes + boolean bloomFilterIndexesRead = false; + HFileBlock block; + while ((block = blockIter.nextBlock()) != null) { + bloomFilterIndexesRead = true; + readNextHeader = readEachBlockInLoadOnOpenDataSection(block, readNextHeader); + } + + reader.close(); + + Assert.assertEquals(hasBloomFilters, bloomFilterIndexesRead); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); + } + + private boolean readEachBlockInLoadOnOpenDataSection(HFileBlock block, boolean readNextHeader) + throws IOException { + long bytesRead = block.getOnDiskSizeWithHeader(); + if (readNextHeader) { + bytesRead -= HFileBlock.headerSize(true); + readNextHeader = false; + } + if (block.getNextBlockOnDiskSize() > 0) { + bytesRead += HFileBlock.headerSize(true); + readNextHeader = true; + } + block.release(); + Assert.assertEquals(bytesRead, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + Assert.assertEquals(1, ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset()); + return readNextHeader; + } + + private void readBloomFilters(Path path, BloomType bt, byte[] key, KeyValue keyValue) + throws IOException { + Assert.assertTrue(keyValue == null || key == null); + + // Assert that the bloom filter index was read and it's size is accounted in bytes read from + // fs + readLoadOnOpenDataSection(path, true); + + CacheConfig cacheConf = new CacheConfig(conf); + StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, path, true); + HStoreFile sf = new HStoreFile(storeFileInfo, bt, cacheConf); + + // Read HFile trailer and load-on-open data section + sf.initReader(); + + // Reset bytes read from fs to 0 + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + // Reset read ops count to 0 + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); + + StoreFileReader reader = sf.getReader(); + BloomFilter bloomFilter = reader.getGeneralBloomFilter(); + Assert.assertTrue(bloomFilter instanceof CompoundBloomFilter); + CompoundBloomFilter cbf = (CompoundBloomFilter) bloomFilter; + + // Get the bloom filter index reader + HFileBlockIndex.BlockIndexReader index = cbf.getBloomIndex(); + int block; + + // Search for the key in the bloom filter index + if (keyValue != null) { + block = index.rootBlockContainingKey(keyValue); + } else { + byte[] row = key; + block = index.rootBlockContainingKey(row, 0, row.length); + } + + // Read the bloom block from FS + HFileBlock bloomBlock = cbf.getBloomBlock(block); + long bytesRead = bloomBlock.getOnDiskSizeWithHeader(); + if (bloomBlock.getNextBlockOnDiskSize() > 0) { + bytesRead += HFileBlock.headerSize(true); + } + // Asser that the block read is a bloom block + Assert.assertEquals(bloomBlock.getBlockType(), BlockType.BLOOM_CHUNK); + bloomBlock.release(); + + // Close the reader + reader.close(true); + + Assert.assertEquals(bytesRead, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + Assert.assertEquals(1, ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset()); + } + + private void writeBloomFilters(Path path, BloomType bt, int bloomBlockByteSize) + throws IOException { + conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, bloomBlockByteSize); + CacheConfig cacheConf = new CacheConfig(conf); + HFileContext meta = new HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE) + .withIncludesTags(false).withDataBlockEncoding(DataBlockEncoding.NONE) + .withCompression(Compression.Algorithm.NONE).build(); + StoreFileWriter w = new StoreFileWriter.Builder(conf, cacheConf, fs).withFileContext(meta) + .withBloomType(bt).withFilePath(path).build(); + Assert.assertTrue(w.hasGeneralBloom()); + Assert.assertTrue(w.getGeneralBloomWriter() instanceof CompoundBloomFilterWriter); + CompoundBloomFilterWriter cbbf = (CompoundBloomFilterWriter) w.getGeneralBloomWriter(); + byte[] cf = Bytes.toBytes("cf"); + byte[] cq = Bytes.toBytes("cq"); + for (int i = 0; i < NUM_KEYS; i++) { + byte[] keyBytes = RandomKeyValueUtil.randomOrderedFixedLengthKey(RNG, i, 10); + // A random-length random value. + byte[] valueBytes = RandomKeyValueUtil.randomFixedLengthValue(RNG, 10); + KeyValue keyValue = + new KeyValue(keyBytes, cf, cq, EnvironmentEdgeManager.currentTime(), valueBytes); + w.append(keyValue); + keyList.add(keyBytes); + keyValues.add(keyValue); + } + Assert.assertEquals(keyList.size(), cbbf.getKeyCount()); + w.close(); + } + + private static class MyNoOpEncodedSeeker extends NoOpIndexBlockEncoder.NoOpEncodedSeeker { + public long getBlockOffset(int i) { + return blockOffsets[i]; + } + + public int getBlockDataSize(int i) { + return blockDataSizes[i]; + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 189af113b334..66e2db5e67c6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.IOTests; @@ -201,6 +202,90 @@ public void testReaderWithLRUBlockCache() throws Exception { lru.shutdown(); } + private void assertBytesReadFromCache(boolean isScanMetricsEnabled) throws Exception { + assertBytesReadFromCache(isScanMetricsEnabled, DataBlockEncoding.NONE); + } + + private void assertBytesReadFromCache(boolean isScanMetricsEnabled, DataBlockEncoding encoding) + throws Exception { + // Write a store file + Path storeFilePath = writeStoreFile(); + + // Initialize the block cache and HFile reader + BlockCache lru = BlockCacheFactory.createBlockCache(conf); + Assert.assertTrue(lru instanceof LruBlockCache); + CacheConfig cacheConfig = new CacheConfig(conf, null, lru, ByteBuffAllocator.HEAP); + HFileReaderImpl reader = + (HFileReaderImpl) HFile.createReader(fs, storeFilePath, cacheConfig, true, conf); + + // Read the first block in HFile from the block cache. + final int offset = 0; + BlockCacheKey cacheKey = new BlockCacheKey(storeFilePath.getName(), offset); + HFileBlock block = (HFileBlock) lru.getBlock(cacheKey, false, false, true); + Assert.assertNull(block); + + // Assert that first block has not been cached in the block cache and no disk I/O happened to + // check that. + ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset(); + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, null); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + + // Read the first block from the HFile. + block = reader.readBlock(offset, -1, true, true, false, true, BlockType.DATA, null); + Assert.assertNotNull(block); + int bytesReadFromFs = block.getOnDiskSizeWithHeader(); + if (block.getNextBlockOnDiskSize() > 0) { + bytesReadFromFs += block.headerSize(); + } + block.release(); + // Assert that disk I/O happened to read the first block. + Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromFs : 0, + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); + + // Read the first block again and assert that it has been cached in the block cache. + block = reader.getCachedBlock(cacheKey, false, false, true, BlockType.DATA, encoding); + long bytesReadFromCache = 0; + if (encoding == DataBlockEncoding.NONE) { + Assert.assertNotNull(block); + bytesReadFromCache = block.getOnDiskSizeWithHeader(); + if (block.getNextBlockOnDiskSize() > 0) { + bytesReadFromCache += block.headerSize(); + } + block.release(); + // Assert that bytes read from block cache account for same number of bytes that would have + // been read from FS if block cache wasn't there. + Assert.assertEquals(bytesReadFromFs, bytesReadFromCache); + } else { + Assert.assertNull(block); + } + Assert.assertEquals(isScanMetricsEnabled ? bytesReadFromCache : 0, + ThreadLocalServerSideScanMetrics.getBytesReadFromBlockCacheAndReset()); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset()); + + reader.close(); + } + + @Test + public void testBytesReadFromCache() throws Exception { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + assertBytesReadFromCache(true); + } + + @Test + public void testBytesReadFromCacheWithScanMetricsDisabled() throws Exception { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false); + assertBytesReadFromCache(false); + } + + @Test + public void testBytesReadFromCacheWithInvalidDataEncoding() throws Exception { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + assertBytesReadFromCache(true, DataBlockEncoding.FAST_DIFF); + } + private BlockCache initCombinedBlockCache(final String l1CachePolicy) { Configuration that = HBaseConfiguration.create(conf); that.setFloat(BUCKET_CACHE_SIZE_KEY, 32); // 32MB for bucket cache. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBytesReadServerSideScanMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBytesReadServerSideScanMetrics.java new file mode 100644 index 000000000000..ff4b63e399ce --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBytesReadServerSideScanMetrics.java @@ -0,0 +1,894 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.TreeSet; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Consumer; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.client.metrics.ScanMetrics; +import org.apache.hadoop.hbase.executor.ExecutorType; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.CompoundBloomFilter; +import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.io.hfile.NoOpIndexBlockEncoder; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.BloomFilter; +import org.apache.hadoop.hbase.util.BloomFilterUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ IOTests.class, LargeTests.class }) +public class TestBytesReadServerSideScanMetrics { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBytesReadServerSideScanMetrics.class); + + @Rule + public TestName name = new TestName(); + + private static final Logger LOG = + LoggerFactory.getLogger(TestBytesReadServerSideScanMetrics.class); + + private HBaseTestingUtility UTIL; + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final byte[] VALUE = Bytes.toBytes("value"); + + private static final byte[] ROW2 = Bytes.toBytes("row2"); + private static final byte[] ROW3 = Bytes.toBytes("row3"); + private static final byte[] ROW4 = Bytes.toBytes("row4"); + + private Configuration conf; + + @Before + public void setUp() throws Exception { + UTIL = new HBaseTestingUtility(); + conf = UTIL.getConfiguration(); + conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0); + conf.setBoolean(CompactSplit.HBASE_REGION_SERVER_ENABLE_COMPACTION, false); + } + + @Test + public void testScanMetricsDisabled() throws Exception { + conf.setInt(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0); + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, false, BloomType.NONE); + writeData(tableName, true); + Scan scan = new Scan(); + scan.withStartRow(ROW2, true); + scan.withStopRow(ROW4, true); + scan.setCaching(1); + try (Table table = UTIL.getConnection().getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(2, rowCount); + Assert.assertNull(scanner.getScanMetrics()); + } + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadFromFsForSerialSeeks() throws Exception { + conf.setInt(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0); + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, false, BloomType.ROW); + writeData(tableName, true); + ScanMetrics scanMetrics = readDataAndGetScanMetrics(tableName, true); + + // Use oldest timestamp to make sure the fake key is not less than the first key in + // the file containing key: row2 + KeyValue keyValue = new KeyValue(ROW2, CF, CQ, HConstants.OLDEST_TIMESTAMP, VALUE); + assertBytesReadFromFs(tableName, scanMetrics.bytesReadFromFs.get(), keyValue, + scanMetrics.blockReadOpsCount.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadFromFsForParallelSeeks() throws Exception { + conf.setInt(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0); + // This property doesn't work correctly if only applied at column family level. + conf.setBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, true); + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, false, BloomType.NONE); + writeData(tableName, true); + HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(0); + ThreadPoolExecutor executor = + server.getExecutorService().getExecutorThreadPool(ExecutorType.RS_PARALLEL_SEEK); + long tasksCompletedBeforeRead = executor.getCompletedTaskCount(); + ScanMetrics scanMetrics = readDataAndGetScanMetrics(tableName, true); + long tasksCompletedAfterRead = executor.getCompletedTaskCount(); + // Assert both of the HFiles were read using parallel seek executor + Assert.assertEquals(2, tasksCompletedAfterRead - tasksCompletedBeforeRead); + + // Use oldest timestamp to make sure the fake key is not less than the first key in + // the file containing key: row2 + KeyValue keyValue = new KeyValue(ROW2, CF, CQ, HConstants.OLDEST_TIMESTAMP, VALUE); + assertBytesReadFromFs(tableName, scanMetrics.bytesReadFromFs.get(), keyValue, + scanMetrics.blockReadOpsCount.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadFromBlockCache() throws Exception { + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, true, BloomType.NONE); + HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(0); + LruBlockCache blockCache = (LruBlockCache) server.getBlockCache().get(); + + // Assert that acceptable size of LRU block cache is greater than 1MB + Assert.assertTrue(blockCache.acceptableSize() > 1024 * 1024); + writeData(tableName, true); + readDataAndGetScanMetrics(tableName, false); + KeyValue keyValue = new KeyValue(ROW2, CF, CQ, HConstants.OLDEST_TIMESTAMP, VALUE); + assertBlockCacheWarmUp(tableName, keyValue); + ScanMetrics scanMetrics = readDataAndGetScanMetrics(tableName, true); + Assert.assertEquals(0, scanMetrics.bytesReadFromFs.get()); + assertBytesReadFromBlockCache(tableName, scanMetrics.bytesReadFromBlockCache.get(), keyValue); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadFromMemstore() throws Exception { + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, false, BloomType.NONE); + writeData(tableName, false); + ScanMetrics scanMetrics = readDataAndGetScanMetrics(tableName, true); + + // Assert no flush has happened for the table + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + HStore store = region.getStore(CF); + // Assert no HFile is there + Assert.assertEquals(0, store.getStorefiles().size()); + } + + KeyValue keyValue = new KeyValue(ROW2, CF, CQ, HConstants.LATEST_TIMESTAMP, VALUE); + int singleKeyValueSize = Segment.getCellLength(keyValue); + // First key value will be read on doing seek and second one on doing next() to determine + // there are no more cells in the row. We don't count key values read on SegmentScanner + // instance creation. + int totalKeyValueSize = 2 * singleKeyValueSize; + Assert.assertEquals(0, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(totalKeyValueSize, scanMetrics.bytesReadFromMemstore.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadWithSwitchFromPReadToStream() throws Exception { + // Set pread max bytes to 3 to make sure that the first row is read using pread and the second + // one using stream read + Map configuration = new HashMap<>(); + configuration.put(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, "3"); + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, true, BloomType.ROW, configuration); + writeData(tableName, true); + Scan scan = new Scan(); + scan.withStartRow(ROW2, true); + scan.withStopRow(ROW4, true); + scan.setScanMetricsEnabled(true); + // Set caching to 1 so that one row is read via PREAD and other via STREAM + scan.setCaching(1); + ScanMetrics scanMetrics = null; + StoreScanner.instrument(); + try (Table table = UTIL.getConnection().getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + Assert.assertFalse(StoreScanner.hasSwitchedToStreamRead()); + for (Result r : scanner) { + rowCount++; + } + Assert.assertTrue(StoreScanner.hasSwitchedToStreamRead()); + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + int bytesReadFromFs = getBytesReadFromFsForNonGetScan(tableName, scanMetrics, 2); + Assert.assertEquals(bytesReadFromFs, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + // There are 2 HFiles so, 1 read op per HFile was done by actual scan to read data block. + // No bloom blocks will be read as this is non Get scan and only bloom filter type is ROW. + Assert.assertEquals(2, scanMetrics.blockReadOpsCount.get()); + // With scan caching set to 1 and 2 rows being scanned, 2 RPC calls will be needed. + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadWhenFlushHappenedInTheMiddleOfScan() throws Exception { + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, true, BloomType.ROW); + writeData(tableName, false); + Scan scan = new Scan(); + scan.withStartRow(ROW2, true); + scan.withStopRow(ROW4, true); + scan.setScanMetricsEnabled(true); + // Set caching to 1 so that one row is read per RPC call + scan.setCaching(1); + ScanMetrics scanMetrics = null; + try (Table table = UTIL.getConnection().getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + if (rowCount == 1) { + flushAndWaitUntilFlushed(tableName, true); + } + } + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + + // Only 1 HFile will be created and it will have only one data block. + int bytesReadFromFs = getBytesReadFromFsForNonGetScan(tableName, scanMetrics, 1); + Assert.assertEquals(bytesReadFromFs, scanMetrics.bytesReadFromFs.get()); + + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + + // Flush happens after first row is returned from server but before second row is returned. + // So, 2 cells will be read from memstore i.e. the cell for the first row and the next cell + // at which scanning will stop. Per row we have 1 cell. + int bytesReadFromMemstore = + Segment.getCellLength(new KeyValue(ROW2, CF, CQ, HConstants.LATEST_TIMESTAMP, VALUE)); + Assert.assertEquals(2 * bytesReadFromMemstore, scanMetrics.bytesReadFromMemstore.get()); + + // There will be 1 read op to read the only data block present in the HFile. + Assert.assertEquals(1, scanMetrics.blockReadOpsCount.get()); + + // More than 1 RPC call should be there + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadInReverseScan() throws Exception { + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, true, BloomType.ROW); + writeData(tableName, true); + Scan scan = new Scan(); + scan.withStartRow(ROW4, true); + scan.withStopRow(ROW2, true); + scan.setScanMetricsEnabled(true); + scan.setReversed(true); + // Set caching to 1 so that one row is read per RPC call + scan.setCaching(1); + ScanMetrics scanMetrics = null; + try (Table table = UTIL.getConnection().getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + System.out.println("Scan metrics: " + scanMetrics.toString()); + } + + // 1 data block per HFile was read. + int bytesReadFromFs = getBytesReadFromFsForNonGetScan(tableName, scanMetrics, 2); + Assert.assertEquals(bytesReadFromFs, scanMetrics.bytesReadFromFs.get()); + + // For the HFile containing both the rows, the data block will be read from block cache when + // KeyValueHeap.next() will be called to read the second row. + // KeyValueHeap.next() will call StoreFileScanner.next() when on ROW4 which is last row of the + // file causing curBlock to be set to null in underlying HFileScanner. As curBlock is null, + // kvNext will be null and call to StoreFileScanner.seekToPreviousRow() will be made. As the + // curBlock of HFileScanner is null so, StoreFileScanner.seekToPreviousRow() will load data + // block from BlockCache. So, 1 data block will be read from block cache. + Assert.assertEquals(bytesReadFromFs / 2, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + + // 1 read op per HFile was done by actual scan to read data block. + Assert.assertEquals(2, scanMetrics.blockReadOpsCount.get()); + + // 2 RPC calls will be there + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + } finally { + UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBytesReadWithLazySeek() throws Exception { + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + createTable(tableName, true, BloomType.NONE); + writeData(tableName, true); + try (Table table = UTIL.getConnection().getTable(tableName)) { + byte[] newValue = Bytes.toBytes("new value"); + // Update the value of ROW2 and let it stay in memstore. Will assert that lazy seek doesn't + // lead to seek on the HFile. + table.put(new Put(ROW2).addColumn(CF, CQ, newValue)); + Scan scan = new Scan(); + scan.withStartRow(ROW2, true); + scan.withStopRow(ROW2, true); + scan.setScanMetricsEnabled(true); + Map> familyMap = new HashMap<>(); + familyMap.put(CF, new TreeSet<>(Bytes.BYTES_COMPARATOR)); + familyMap.get(CF).add(CQ); + scan.setFamilyMap(familyMap); + ScanMetrics scanMetrics = null; + try (ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + Assert.assertArrayEquals(newValue, r.getValue(CF, CQ)); + } + Assert.assertEquals(1, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + // No real seek should be done on the HFile. + Assert.assertEquals(0, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.blockReadOpsCount.get()); + + // The cell should be coming purely from memstore. + int cellSize = + Segment.getCellLength(new KeyValue(ROW2, CF, CQ, HConstants.LATEST_TIMESTAMP, newValue)); + Assert.assertEquals(cellSize, scanMetrics.bytesReadFromMemstore.get()); + Assert.assertEquals(1, scanMetrics.countOfRPCcalls.get()); + } + } finally { + UTIL.shutdownMiniCluster(); + } + } + + /** + * Test consecutive calls to RegionScannerImpl.next() to make sure populating scan metrics from + * ThreadLocalServerSideScanMetrics is done correctly. + */ + @Test + public void testConsecutiveRegionScannerNextCalls() throws Exception { + // We will be setting a very small block size so, make sure to set big enough pread max bytes + Map configuration = new HashMap<>(); + configuration.put(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, Integer.toString(64 * 1024)); + UTIL.startMiniCluster(); + try { + TableName tableName = TableName.valueOf(name.getMethodName()); + // Set the block size to 4 bytes to get 1 row per data block in HFile. + createTable(tableName, true, BloomType.NONE, 4, configuration); + try (Table table = UTIL.getConnection().getTable(tableName)) { + // Add 3 rows to the table. + table.put(new Put(ROW2).addColumn(CF, CQ, VALUE)); + table.put(new Put(ROW3).addColumn(CF, CQ, VALUE)); + table.put(new Put(ROW4).addColumn(CF, CQ, VALUE)); + + ScanMetrics scanMetrics = null; + + // Scan the added rows. The rows should be read from memstore. + Scan scan = createScanToReadOneRowAtATimeFromServer(ROW2, ROW3); + try (ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + + // Assert that rows were read from only memstore and involved 2 RPC calls. + int cellSize = + Segment.getCellLength(new KeyValue(ROW2, CF, CQ, HConstants.LATEST_TIMESTAMP, VALUE)); + Assert.assertEquals(0, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.blockReadOpsCount.get()); + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + Assert.assertEquals(3 * cellSize, scanMetrics.bytesReadFromMemstore.get()); + + // Flush the table and make sure that the rows are read from HFiles. + flushAndWaitUntilFlushed(tableName, false); + scan = createScanToReadOneRowAtATimeFromServer(ROW2, ROW3); + scanMetrics = null; + try (ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + + // Assert that rows were read from HFiles and involved 2 RPC calls. + int bytesReadFromFs = getBytesReadToReadConsecutiveDataBlocks(tableName, 1, 3, true); + Assert.assertEquals(bytesReadFromFs, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(3, scanMetrics.blockReadOpsCount.get()); + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + + // Make sure that rows are read from Blockcache now. + scan = createScanToReadOneRowAtATimeFromServer(ROW2, ROW3); + scanMetrics = null; + try (ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(2, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + + // Assert that rows were read from Blockcache and involved 2 RPC calls. + int bytesReadFromBlockCache = + getBytesReadToReadConsecutiveDataBlocks(tableName, 1, 3, false); + Assert.assertEquals(bytesReadFromBlockCache, scanMetrics.bytesReadFromBlockCache.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromFs.get()); + Assert.assertEquals(0, scanMetrics.blockReadOpsCount.get()); + Assert.assertEquals(2, scanMetrics.countOfRPCcalls.get()); + Assert.assertEquals(0, scanMetrics.bytesReadFromMemstore.get()); + } + } finally { + UTIL.shutdownMiniCluster(); + } + } + + private Scan createScanToReadOneRowAtATimeFromServer(byte[] startRow, byte[] stopRow) { + Scan scan = new Scan(); + scan.withStartRow(startRow, true); + scan.withStopRow(stopRow, true); + scan.setScanMetricsEnabled(true); + scan.setCaching(1); + return scan; + } + + private void flushAndWaitUntilFlushed(TableName tableName, boolean waitForUpdatedReaders) + throws Exception { + if (waitForUpdatedReaders) { + StoreScanner.instrument(); + } + UTIL.flush(tableName); + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + HRegion region = regions.get(0); + HStore store = region.getStore(CF); + // In milliseconds + int maxWaitTime = 100000; + int totalWaitTime = 0; + int sleepTime = 10000; + while ( + store.getStorefiles().size() == 0 + || (waitForUpdatedReaders && !StoreScanner.hasUpdatedReaders()) + ) { + Thread.sleep(sleepTime); + totalWaitTime += sleepTime; + if (totalWaitTime >= maxWaitTime) { + throw new Exception("Store files not flushed after " + maxWaitTime + "ms"); + } + } + Assert.assertEquals(1, store.getStorefiles().size()); + } + + private int getBytesReadToReadConsecutiveDataBlocks(TableName tableName, + int expectedStoreFileCount, int expectedDataBlockCount, boolean isReadFromFs) throws Exception { + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + HRegion region = regions.get(0); + HStore store = region.getStore(CF); + Collection storeFiles = store.getStorefiles(); + Assert.assertEquals(expectedStoreFileCount, storeFiles.size()); + int bytesReadFromFs = 0; + for (HStoreFile storeFile : storeFiles) { + StoreFileReader reader = storeFile.getReader(); + HFile.Reader hfileReader = reader.getHFileReader(); + HFileBlock.FSReader blockReader = hfileReader.getUncachedBlockReader(); + FixedFileTrailer trailer = hfileReader.getTrailer(); + int dataIndexLevels = trailer.getNumDataIndexLevels(); + long loadOnOpenDataOffset = trailer.getLoadOnOpenDataOffset(); + HFileBlock.BlockIterator blockIterator = blockReader.blockRange(0, loadOnOpenDataOffset); + HFileBlock block; + boolean readNextBlock = false; + int blockCount = 0; + while ((block = blockIterator.nextBlock()) != null) { + blockCount++; + bytesReadFromFs += block.getOnDiskSizeWithHeader(); + if (isReadFromFs && readNextBlock) { + // This accounts for savings we get from prefetched header but these saving are only + // applicable when reading from FS and not from BlockCache. + bytesReadFromFs -= block.headerSize(); + readNextBlock = false; + } + if (block.getNextBlockOnDiskSize() > 0) { + bytesReadFromFs += block.headerSize(); + readNextBlock = true; + } + Assert.assertTrue(block.getBlockType().isData()); + } + blockIterator.freeBlocks(); + // No intermediate or leaf index blocks are expected. + Assert.assertEquals(1, dataIndexLevels); + Assert.assertEquals(expectedDataBlockCount, blockCount); + } + return bytesReadFromFs; + } + + private int getBytesReadFromFsForNonGetScan(TableName tableName, ScanMetrics scanMetrics, + int expectedStoreFileCount) throws Exception { + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + HRegion region = regions.get(0); + HStore store = region.getStore(CF); + Collection storeFiles = store.getStorefiles(); + Assert.assertEquals(expectedStoreFileCount, storeFiles.size()); + int bytesReadFromFs = 0; + for (HStoreFile storeFile : storeFiles) { + StoreFileReader reader = storeFile.getReader(); + HFile.Reader hfileReader = reader.getHFileReader(); + HFileBlock.FSReader blockReader = hfileReader.getUncachedBlockReader(); + FixedFileTrailer trailer = hfileReader.getTrailer(); + int dataIndexLevels = trailer.getNumDataIndexLevels(); + // Read the first block of the HFile. First block is always expected to be a DATA block and + // the HFile is expected to have only one DATA block. + HFileBlock block = blockReader.readBlockData(0, -1, true, true, true); + Assert.assertTrue(block.getBlockType().isData()); + bytesReadFromFs += block.getOnDiskSizeWithHeader(); + if (block.getNextBlockOnDiskSize() > 0) { + bytesReadFromFs += block.headerSize(); + } + block.release(); + // Each of the HFiles is expected to have only root index but no intermediate or leaf index + // blocks. + Assert.assertEquals(1, dataIndexLevels); + } + return bytesReadFromFs; + } + + private ScanMetrics readDataAndGetScanMetrics(TableName tableName, boolean isScanMetricsEnabled) + throws Exception { + Scan scan = new Scan(); + scan.withStartRow(ROW2, true); + scan.withStopRow(ROW2, true); + scan.setScanMetricsEnabled(isScanMetricsEnabled); + ScanMetrics scanMetrics; + try (Table table = UTIL.getConnection().getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + int rowCount = 0; + StoreFileScanner.instrument(); + for (Result r : scanner) { + rowCount++; + } + Assert.assertEquals(1, rowCount); + scanMetrics = scanner.getScanMetrics(); + } + if (isScanMetricsEnabled) { + LOG.info("Bytes read from fs: " + scanMetrics.bytesReadFromFs.get()); + LOG.info("Bytes read from block cache: " + scanMetrics.bytesReadFromBlockCache.get()); + LOG.info("Bytes read from memstore: " + scanMetrics.bytesReadFromMemstore.get()); + LOG.info("Count of bytes scanned: " + scanMetrics.countOfBlockBytesScanned.get()); + LOG.info("StoreFileScanners seek count: " + StoreFileScanner.getSeekCount()); + } + return scanMetrics; + } + + private void writeData(TableName tableName, boolean shouldFlush) throws Exception { + try (Table table = UTIL.getConnection().getTable(tableName)) { + table.put(new Put(ROW2).addColumn(CF, CQ, VALUE)); + table.put(new Put(ROW4).addColumn(CF, CQ, VALUE)); + if (shouldFlush) { + // Create a HFile + UTIL.flush(tableName); + } + + table.put(new Put(Bytes.toBytes("row1")).addColumn(CF, CQ, VALUE)); + table.put(new Put(Bytes.toBytes("row5")).addColumn(CF, CQ, VALUE)); + if (shouldFlush) { + // Create a HFile + UTIL.flush(tableName); + } + } + } + + private void createTable(TableName tableName, boolean blockCacheEnabled, BloomType bloomType) + throws Exception { + createTable(tableName, blockCacheEnabled, bloomType, HConstants.DEFAULT_BLOCKSIZE, + new HashMap<>()); + } + + private void createTable(TableName tableName, boolean blockCacheEnabled, BloomType bloomType, + Map configuration) throws Exception { + createTable(tableName, blockCacheEnabled, bloomType, HConstants.DEFAULT_BLOCKSIZE, + configuration); + } + + private void createTable(TableName tableName, boolean blockCacheEnabled, BloomType bloomType, + int blocksize, Map configuration) throws Exception { + Admin admin = UTIL.getAdmin(); + TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); + ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder = + ColumnFamilyDescriptorBuilder.newBuilder(CF); + columnFamilyDescriptorBuilder.setBloomFilterType(bloomType); + columnFamilyDescriptorBuilder.setBlockCacheEnabled(blockCacheEnabled); + columnFamilyDescriptorBuilder.setBlocksize(blocksize); + for (Map.Entry entry : configuration.entrySet()) { + columnFamilyDescriptorBuilder.setConfiguration(entry.getKey(), entry.getValue()); + } + tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptorBuilder.build()); + admin.createTable(tableDescriptorBuilder.build()); + UTIL.waitUntilAllRegionsAssigned(tableName); + } + + private void assertBytesReadFromFs(TableName tableName, long actualBytesReadFromFs, + KeyValue keyValue, long actualReadOps) throws Exception { + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + MutableInt totalExpectedBytesReadFromFs = new MutableInt(0); + MutableInt totalExpectedReadOps = new MutableInt(0); + for (HRegion region : regions) { + Assert.assertNull(region.getBlockCache()); + HStore store = region.getStore(CF); + Collection storeFiles = store.getStorefiles(); + Assert.assertEquals(2, storeFiles.size()); + for (HStoreFile storeFile : storeFiles) { + StoreFileReader reader = storeFile.getReader(); + HFile.Reader hfileReader = reader.getHFileReader(); + BloomFilter bloomFilter = reader.getGeneralBloomFilter(); + Assert.assertTrue(bloomFilter == null || bloomFilter instanceof CompoundBloomFilter); + CompoundBloomFilter cbf = bloomFilter == null ? null : (CompoundBloomFilter) bloomFilter; + Consumer bytesReadFunction = new Consumer() { + @Override + public void accept(HFileBlock block) { + totalExpectedBytesReadFromFs.add(block.getOnDiskSizeWithHeader()); + if (block.getNextBlockOnDiskSize() > 0) { + totalExpectedBytesReadFromFs.add(block.headerSize()); + } + totalExpectedReadOps.add(1); + } + }; + readHFile(hfileReader, cbf, keyValue, bytesReadFunction); + } + } + Assert.assertEquals(totalExpectedBytesReadFromFs.longValue(), actualBytesReadFromFs); + Assert.assertEquals(totalExpectedReadOps.longValue(), actualReadOps); + } + + private void readHFile(HFile.Reader hfileReader, CompoundBloomFilter cbf, KeyValue keyValue, + Consumer bytesReadFunction) throws Exception { + HFileBlock.FSReader blockReader = hfileReader.getUncachedBlockReader(); + FixedFileTrailer trailer = hfileReader.getTrailer(); + HFileContext meta = hfileReader.getFileContext(); + long fileSize = hfileReader.length(); + + // Read the bloom block from FS + if (cbf != null) { + // Read a block in load-on-open section to make sure prefetched header is not bloom + // block's header + blockReader.readBlockData(trailer.getLoadOnOpenDataOffset(), -1, true, true, true).release(); + + HFileBlockIndex.BlockIndexReader index = cbf.getBloomIndex(); + byte[] row = ROW2; + int blockIndex = index.rootBlockContainingKey(row, 0, row.length); + HFileBlock bloomBlock = cbf.getBloomBlock(blockIndex); + boolean fileContainsKey = BloomFilterUtil.contains(row, 0, row.length, + bloomBlock.getBufferReadOnly(), bloomBlock.headerSize(), + bloomBlock.getUncompressedSizeWithoutHeader(), cbf.getHash(), cbf.getHashCount()); + bytesReadFunction.accept(bloomBlock); + // Asser that the block read is a bloom block + Assert.assertEquals(bloomBlock.getBlockType(), BlockType.BLOOM_CHUNK); + bloomBlock.release(); + if (!fileContainsKey) { + // Key is not in th file, so we don't need to read the data block + return; + } + } + + // Indexes use NoOpEncodedSeeker + MyNoOpEncodedSeeker seeker = new MyNoOpEncodedSeeker(); + HFileBlock.BlockIterator blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), + fileSize - trailer.getTrailerSize()); + HFileBlock block = blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX); + + // Comparator class name is stored in the trailer in version 3. + CellComparator comparator = trailer.createComparator(); + // Initialize the seeker + seeker.initRootIndex(block, trailer.getDataIndexCount(), comparator, + trailer.getNumDataIndexLevels()); + + int blockLevelsRead = 1; // Root index is the first level + + int rootLevIndex = seeker.rootBlockContainingKey(keyValue); + long currentOffset = seeker.getBlockOffset(rootLevIndex); + int currentDataSize = seeker.getBlockDataSize(rootLevIndex); + + HFileBlock prevBlock = null; + do { + prevBlock = block; + block = blockReader.readBlockData(currentOffset, currentDataSize, true, true, true); + HFileBlock unpacked = block.unpack(meta, blockReader); + if (unpacked != block) { + block.release(); + block = unpacked; + } + bytesReadFunction.accept(block); + if (!block.getBlockType().isData()) { + ByteBuff buffer = block.getBufferWithoutHeader(); + // Place the buffer at the correct position + HFileBlockIndex.BlockIndexReader.locateNonRootIndexEntry(buffer, keyValue, comparator); + currentOffset = buffer.getLong(); + currentDataSize = buffer.getInt(); + } + prevBlock.release(); + blockLevelsRead++; + } while (!block.getBlockType().isData()); + block.release(); + blockIter.freeBlocks(); + + Assert.assertEquals(blockLevelsRead, trailer.getNumDataIndexLevels() + 1); + } + + private void assertBytesReadFromBlockCache(TableName tableName, + long actualBytesReadFromBlockCache, KeyValue keyValue) throws Exception { + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + MutableInt totalExpectedBytesReadFromBlockCache = new MutableInt(0); + for (HRegion region : regions) { + Assert.assertNotNull(region.getBlockCache()); + HStore store = region.getStore(CF); + Collection storeFiles = store.getStorefiles(); + Assert.assertEquals(2, storeFiles.size()); + for (HStoreFile storeFile : storeFiles) { + StoreFileReader reader = storeFile.getReader(); + HFile.Reader hfileReader = reader.getHFileReader(); + BloomFilter bloomFilter = reader.getGeneralBloomFilter(); + Assert.assertTrue(bloomFilter == null || bloomFilter instanceof CompoundBloomFilter); + CompoundBloomFilter cbf = bloomFilter == null ? null : (CompoundBloomFilter) bloomFilter; + Consumer bytesReadFunction = new Consumer() { + @Override + public void accept(HFileBlock block) { + totalExpectedBytesReadFromBlockCache.add(block.getOnDiskSizeWithHeader()); + if (block.getNextBlockOnDiskSize() > 0) { + totalExpectedBytesReadFromBlockCache.add(block.headerSize()); + } + } + }; + readHFile(hfileReader, cbf, keyValue, bytesReadFunction); + } + } + Assert.assertEquals(totalExpectedBytesReadFromBlockCache.longValue(), + actualBytesReadFromBlockCache); + } + + private void assertBlockCacheWarmUp(TableName tableName, KeyValue keyValue) throws Exception { + List regions = UTIL.getMiniHBaseCluster().getRegions(tableName); + Assert.assertEquals(1, regions.size()); + for (HRegion region : regions) { + Assert.assertNotNull(region.getBlockCache()); + HStore store = region.getStore(CF); + Collection storeFiles = store.getStorefiles(); + Assert.assertEquals(2, storeFiles.size()); + for (HStoreFile storeFile : storeFiles) { + StoreFileReader reader = storeFile.getReader(); + HFile.Reader hfileReader = reader.getHFileReader(); + BloomFilter bloomFilter = reader.getGeneralBloomFilter(); + Assert.assertTrue(bloomFilter == null || bloomFilter instanceof CompoundBloomFilter); + CompoundBloomFilter cbf = bloomFilter == null ? null : (CompoundBloomFilter) bloomFilter; + Consumer bytesReadFunction = new Consumer() { + @Override + public void accept(HFileBlock block) { + assertBlockIsCached(hfileReader, block, region.getBlockCache()); + } + }; + readHFile(hfileReader, cbf, keyValue, bytesReadFunction); + } + } + } + + private void assertBlockIsCached(HFile.Reader hfileReader, HFileBlock block, + BlockCache blockCache) { + if (blockCache == null) { + return; + } + Path path = hfileReader.getPath(); + BlockCacheKey key = new BlockCacheKey(path, block.getOffset(), true, block.getBlockType()); + HFileBlock cachedBlock = (HFileBlock) blockCache.getBlock(key, true, false, true); + Assert.assertNotNull(cachedBlock); + Assert.assertEquals(block.getOnDiskSizeWithHeader(), cachedBlock.getOnDiskSizeWithHeader()); + Assert.assertEquals(block.getNextBlockOnDiskSize(), cachedBlock.getNextBlockOnDiskSize()); + cachedBlock.release(); + } + + private static class MyNoOpEncodedSeeker extends NoOpIndexBlockEncoder.NoOpEncodedSeeker { + public long getBlockOffset(int i) { + return blockOffsets[i]; + } + + public int getBlockDataSize(int i) { + return blockDataSizes[i]; + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index 60fdf2357759..7ac457750095 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -62,6 +63,7 @@ import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.wal.WALFactory; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -354,6 +356,48 @@ public void testMemstoreConcurrentControl() throws IOException { assertScannerResults(s, new KeyValue[] { kv1, kv2 }); } + private long getBytesReadFromMemstore() throws IOException { + final byte[] f = Bytes.toBytes("family"); + final byte[] q1 = Bytes.toBytes("q1"); + final byte[] v = Bytes.toBytes("value"); + int numKvs = 10; + + MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); + + KeyValue kv; + KeyValue[] kvs = new KeyValue[numKvs]; + long totalCellSize = 0; + for (int i = 0; i < numKvs; i++) { + byte[] row = Bytes.toBytes(i); + kv = new KeyValue(row, f, q1, v); + kv.setSequenceId(w.getWriteNumber()); + memstore.add(kv, null); + kvs[i] = kv; + totalCellSize += Segment.getCellLength(kv); + } + mvcc.completeAndWait(w); + + ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset(); + KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); + assertScannerResults(s, kvs); + return totalCellSize; + } + + @Test + public void testBytesReadFromMemstore() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(true); + long totalCellSize = getBytesReadFromMemstore(); + Assert.assertEquals(totalCellSize, + ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset()); + } + + @Test + public void testBytesReadFromMemstoreWithScanMetricsDisabled() throws IOException { + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(false); + getBytesReadFromMemstore(); + Assert.assertEquals(0, ThreadLocalServerSideScanMetrics.getBytesReadFromMemstoreAndReset()); + } + /** * Regression test for HBASE-2616, HBASE-2670. When we insert a higher-memstoreTS version of a * cell but with the same timestamp, we still need to provide consistent reads for the same From 35b6e5872dc43f3f451a9580de986d13acf39070 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:47:12 +0800 Subject: [PATCH 003/336] HBASE-29450 Bump org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0 (#7152) Bumps org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-version: 3.18.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Duo Zhang (cherry picked from commit 8734f7037858ceb4d8e9eba06074dbf7077b64e2) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 825e76190640..5b3a23f84058 100644 --- a/pom.xml +++ b/pom.xml @@ -578,7 +578,7 @@ 2.8.1 1.15 2.18.0 - 3.17.0 + 3.18.0 3.6.1 1.5.0 3.4.4 From 427f62d12dc74b78f31ecabca2b4d19baf74c800 Mon Sep 17 00:00:00 2001 From: ZHENYU LI <48652137+JHSUYU@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:12:38 -0400 Subject: [PATCH 004/336] HBASE-28589: ServerCall.setResponse swallows IOException and leaves client without response (#7156) When IOException occurs during response creation in ServerCall.setResponse(), the method only logs a warning and sets response to null. This causes client to receive no response or experience connection issues without knowing what went wrong on server side. This patch: - Catches IOException during response creation - Creates an error response to send back to client - Handles the case where even error response creation fails - Adds unit tests to verify the behavior Signed-off by: Duo Zhang Signed-off by: Charles Connell --- .../apache/hadoop/hbase/ipc/ServerCall.java | 27 +++ .../hadoop/hbase/ipc/TestServerCall.java | 174 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java index 8d8d443aa127..db181d6d6f3a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java @@ -339,6 +339,7 @@ public synchronized void setResponse(Message m, final CellScanner cells, Throwab bc = new BufferChain(responseBufs); } catch (IOException e) { RpcServer.LOG.warn("Exception while creating response " + e); + bc = createFallbackErrorResponse(e); } this.response = bc; // Once a response message is created and set to this.response, this Call can be treated as @@ -375,6 +376,32 @@ static void setExceptionResponse(Throwable t, String errorMsg, headerBuilder.setException(exceptionBuilder.build()); } + /* + * Creates a fallback error response when the primary response creation fails. This method is + * invoked as a last resort when an IOException occurs during the normal response creation + * process. It attempts to create a minimal error response containing only the error information, + * without any cell blocks or additional data. The purpose is to ensure that the client receives + * some indication of the failure rather than experiencing a silent connection drop. This provides + * better error handling on the client side. + */ + private BufferChain createFallbackErrorResponse(IOException originalException) { + try { + ResponseHeader.Builder headerBuilder = ResponseHeader.newBuilder(); + headerBuilder.setCallId(this.id); + String responseErrorMsg = + "Failed to create response due to: " + originalException.getMessage(); + setExceptionResponse(originalException, responseErrorMsg, headerBuilder); + Message header = headerBuilder.build(); + ByteBuffer headerBuf = createHeaderAndMessageBytes(null, header, 0, null); + this.isError = true; + return new BufferChain(new ByteBuffer[] { headerBuf }); + } catch (IOException e) { + RpcServer.LOG.error("Failed to create error response for client, connection may be dropped", + e); + return null; + } + } + static ByteBuffer createHeaderAndMessageBytes(Message result, Message header, int cellBlockSize, List cellBlock) throws IOException { // Organize the response as a set of bytebuffers rather than collect it all together inside diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java new file mode 100644 index 000000000000..46239e95859b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.codec.Codec; +import org.apache.hadoop.hbase.io.ByteBuffAllocator; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RPCTests; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService; +import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; + +/** + * Test for ServerCall IOException handling in setResponse method. + */ +@Category({ RPCTests.class, MediumTests.class }) +public class TestServerCall { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestServerCall.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestServerCall.class); + + private Configuration conf; + private NettyServerRpcConnection mockConnection; + private RequestHeader header; + private Message mockParam; + private ByteBuffAllocator mockAllocator; + private CellBlockBuilder mockCellBlockBuilder; + private InetAddress mockAddr; + private BlockingService mockService; + private MethodDescriptor mockMethodDescriptor; + + @Before + public void setUp() throws Exception { + conf = HBaseConfiguration.create(); + mockConnection = mock(NettyServerRpcConnection.class); + header = RequestHeader.newBuilder().setCallId(1).setMethodName("testMethod") + .setRequestParam(true).build(); + mockParam = mock(Message.class); + mockAllocator = mock(ByteBuffAllocator.class); + mockCellBlockBuilder = mock(CellBlockBuilder.class); + mockAddr = mock(InetAddress.class); + + mockMethodDescriptor = + org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.getDescriptor() + .getMethods().get(0); + + mockService = mock(BlockingService.class); + mockConnection.codec = mock(Codec.class); + + when(mockAllocator.isReservoirEnabled()).thenReturn(false); + } + + /** + * Test that when IOException occurs during response creation in setResponse, an error response is + * created and sent to the client instead of leaving the response as null. + */ + @Test + public void testSetResponseWithIOException() throws Exception { + // Create a CellBlockBuilder that throws IOException + CellBlockBuilder failingCellBlockBuilder = mock(CellBlockBuilder.class); + doThrow(new IOException("Test IOException during buildCellBlock")).when(failingCellBlockBuilder) + .buildCellBlock(any(), any(), any()); + + // Create NettyServerCall instance + NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, + mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockAllocator, failingCellBlockBuilder, null); + + // Set a successful response, but CellBlockBuilder will fail + Message mockResponse = mock(Message.class); + CellScanner mockCellScanner = mock(CellScanner.class); + + LOG.info("Testing setResponse with IOException in buildCellBlock"); + call.setResponse(mockResponse, mockCellScanner, null, null); + + // Verify that response is not null and contains error information + BufferChain response = call.getResponse(); + assertNotNull("Response should not be null even when IOException occurs", response); + assertTrue("Call should be marked as error", call.isError); + + // Verify the response buffer is valid + ByteBuffer[] bufs = response.getBuffers(); + assertNotNull("Response buffers should not be null", bufs); + assertTrue("Response should have at least one buffer", bufs.length > 0); + } + + /** + * Test the case where both normal response creation and error response creation fail with + * IOException. + */ + @Test + public void testSetResponseWithDoubleIOException() throws Exception { + + CellBlockBuilder failingCellBlockBuilder = mock(CellBlockBuilder.class); + doThrow(new IOException("Test IOException")).when(failingCellBlockBuilder).buildCellBlock(any(), + any(), any()); + + NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, + mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockAllocator, failingCellBlockBuilder, null); + + Message mockResponse = mock(Message.class); + CellScanner mockCellScanner = mock(CellScanner.class); + + // Even if error response creation might fail, the call should still be marked as error + call.setResponse(mockResponse, mockCellScanner, null, null); + assertTrue("Call should be marked as error", call.isError); + } + + /** + * Test normal response creation to ensure our changes don't break the normal flow. + */ + @Test + public void testSetResponseNormalFlow() throws Exception { + CellBlockBuilder normalCellBlockBuilder = mock(CellBlockBuilder.class); + when(normalCellBlockBuilder.buildCellBlock(any(), any(), any())).thenReturn(null); + + NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, + mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockAllocator, normalCellBlockBuilder, null); + + RPCProtos.CellBlockMeta mockResponse = + RPCProtos.CellBlockMeta.newBuilder().setLength(0).build(); + + LOG.info("Testing normal setResponse flow"); + call.setResponse(mockResponse, null, null, null); + + BufferChain response = call.getResponse(); + assertNotNull("Response should not be null in normal flow", response); + assertTrue("Call should not be marked as error in normal flow", !call.isError); + } +} From 90b9aa376052dea44a974709911a8dfc7fbd6639 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Sat, 26 Jul 2025 15:15:47 +0900 Subject: [PATCH 005/336] HBASE-29472 Fix splitting algorithms of RegionSplitter tool (#7173) Signed-off-by: Nihal Jain --- .../hadoop/hbase/util/RegionSplitter.java | 3 ++ .../hadoop/hbase/util/TestRegionSplitter.java | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java index b38f97af8804..ef63985a2d23 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java @@ -978,6 +978,9 @@ public void setLastRow(byte[] userInput) { * @return the midpoint of the 2 numbers */ public BigInteger split2(BigInteger a, BigInteger b) { + if (b.equals(lastRowInt)) { + b = b.add(BigInteger.ONE); + } return a.add(b).divide(BigInteger.valueOf(2)).abs(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java index ff35b059ce88..c97cbd02fbae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.conf.Configuration; @@ -108,6 +109,35 @@ public void testCreatePresplitTableHex() throws Exception { TableName.valueOf(name.getMethodName())); } + /** + * Test creating a pre-split table and splitting it again using the HexStringSplit and + * DecimalStringSplit algorithms. + */ + @Test + public void testSplitPresplitTable() throws Exception { + testSplitPresplitTable(new HexStringSplit()); + testSplitPresplitTable(new DecimalStringSplit()); + } + + private void testSplitPresplitTable(RegionSplitter.NumberStringSplit splitter) throws Exception { + final List initialBounds = new ArrayList<>(); + initialBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + initialBounds.addAll(Arrays.asList(splitter.split(8))); + initialBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + + // Do table creation/pre-splitting and verification of region boundaries + final String className = splitter.getClass().getSimpleName(); + final TableName tableName = TableName.valueOf(className); + preSplitTableAndVerify(initialBounds, className, tableName); + + // Split the table again and verify the new region boundaries + final List expectedBounds = new ArrayList<>(); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + expectedBounds.addAll(Arrays.asList(splitter.split(16))); + expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY); + rollingSplitAndVerify(tableName, className, expectedBounds); + } + /** * Test creating a pre-split table using the UniformSplit algorithm. */ From e1e37d507377e4d7aa16b8ae7bdb70289cbe5ca4 Mon Sep 17 00:00:00 2001 From: Jialun peng <87167182+pjl1070048431@users.noreply.github.com> Date: Sun, 27 Jul 2025 21:37:26 +0800 Subject: [PATCH 006/336] HBASE-29467 Redundant conditions in CostFunction.scale() method (#7170) The first if-block already covers all these cases, making the second if-block completely redundant as it will never be reached with conditions that would make it evaluate differently. Signed-off-by: Duo Zhang Reviewied-by: Kevin Geiszler (cherry picked from commit d76bbe21cf0dfc37f740718e826b8c7ccf582c11) --- .../org/apache/hadoop/hbase/master/balancer/CostFunction.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java index 1dcd4580b1a6..1703d6f7a7e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java @@ -111,10 +111,6 @@ protected static double scale(double min, double max, double value) { ) { return 0; } - if (max <= min || Math.abs(max - min) <= costEpsilon) { - return 0; - } - return Math.max(0d, Math.min(1d, (value - min) / (max - min))); } } From 0e5d044911ed49593db7cca76a48be1f5c14e828 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 28 Jul 2025 22:34:45 +0900 Subject: [PATCH 007/336] HBASE-29474 RegionSplitter.rollingSplit is broken (#7174) Avoid concurrent modification by iterating over a snapshot of the keys. Also, revive the sorting logic for the ServerName list, which was mistakenly removed in 5e91b45b166cd5a68457234f0a62ca1c2b5d9211. Signed-off-by: Duo Zhang --- .../org/apache/hadoop/hbase/util/RegionSplitter.java | 12 +++++++----- .../apache/hadoop/hbase/util/TestRegionSplitter.java | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java index ef63985a2d23..5feee61bee00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java @@ -21,9 +21,9 @@ import java.math.BigInteger; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TreeMap; import org.apache.commons.lang3.ArrayUtils; @@ -459,13 +459,15 @@ static void rollingSplit(TableName tableName, SplitAlgorithm splitAlgo, Configur } } + // Sort the ServerNames by the number of regions they have + final List serversLeft = Lists.newArrayList(daughterRegions.keySet()); + serversLeft.sort(Comparator.comparing(rsSizes::get)); + // Round-robin through the ServerName list. Choose the lightest-loaded servers // first to keep the master from load-balancing regions as we split. - for (Map.Entry>> daughterRegion : daughterRegions.entrySet()) { + for (final ServerName rsLoc : serversLeft) { Pair dr = null; - ServerName rsLoc = daughterRegion.getKey(); - LinkedList> regionList = daughterRegion.getValue(); + final LinkedList> regionList = daughterRegions.get(rsLoc); // Find a region in the ServerName list that hasn't been moved LOG.debug("Finding a region on " + rsLoc); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java index c97cbd02fbae..d45d95732b86 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java @@ -72,7 +72,7 @@ public class TestRegionSplitter { @BeforeClass public static void setup() throws Exception { - UTIL.startMiniCluster(); + UTIL.startMiniCluster(2); } @AfterClass From 8d30fabb5de3404f87a266f176481fa1bd88a73e Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Mon, 28 Jul 2025 15:20:54 -0400 Subject: [PATCH 008/336] HBASE-29447 Fix WAL archives cause incremental backup failures (#7151) (#7160) (#7164) Signed-off-by: Ray Mattingly Co-authored-by: Hernan Romer Co-authored-by: Hernan Gelaf-Romer --- .../hbase/mapreduce/WALInputFormat.java | 34 ++++++++++-- .../hbase/mapreduce/TestWALInputFormat.java | 55 ++++++++++++++++++- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 7362f585d319..03d3250f54a9 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -318,7 +318,7 @@ List getSplits(final JobContext context, final String startKey, fina for (Path inputPath : inputPaths) { FileSystem fs = inputPath.getFileSystem(conf); try { - List files = getFiles(fs, inputPath, startTime, endTime); + List files = getFiles(fs, inputPath, startTime, endTime, conf); allFiles.addAll(files); } catch (FileNotFoundException e) { if (ignoreMissing) { @@ -349,11 +349,11 @@ private Path[] getInputPaths(Configuration conf) { * equal to this value else we will filter out the file. If name does not seem to * have a timestamp, we will just return it w/o filtering. */ - private List getFiles(FileSystem fs, Path dir, long startTime, long endTime) - throws IOException { + private List getFiles(FileSystem fs, Path dir, long startTime, long endTime, + Configuration conf) throws IOException { List result = new ArrayList<>(); LOG.debug("Scanning " + dir.toString() + " for WAL files"); - RemoteIterator iter = fs.listLocatedStatus(dir); + RemoteIterator iter = listLocatedFileStatus(fs, dir, conf); if (!iter.hasNext()) { return Collections.emptyList(); } @@ -361,7 +361,7 @@ private List getFiles(FileSystem fs, Path dir, long startTime, long LocatedFileStatus file = iter.next(); if (file.isDirectory()) { // Recurse into sub directories - result.addAll(getFiles(fs, file.getPath(), startTime, endTime)); + result.addAll(getFiles(fs, file.getPath(), startTime, endTime, conf)); } else { addFile(result, file, startTime, endTime); } @@ -396,4 +396,28 @@ public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { return new WALKeyRecordReader(); } + + /** + * Attempts to return the {@link LocatedFileStatus} for the given directory. If the directory does + * not exist, it will check if the directory is an archived log file and try to find it + */ + private static RemoteIterator listLocatedFileStatus(FileSystem fs, Path dir, + Configuration conf) throws IOException { + try { + return fs.listLocatedStatus(dir); + } catch (FileNotFoundException e) { + if (AbstractFSWALProvider.isArchivedLogFile(dir)) { + throw e; + } + + LOG.warn("Log file {} not found, trying to find it in archive directory.", dir); + Path archiveFile = AbstractFSWALProvider.findArchivedLog(dir, conf); + if (archiveFile == null) { + LOG.error("Did not find archive file for {}", dir); + throw e; + } + + return fs.listLocatedStatus(archiveFile); + } + } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java index 70602a371668..6fdfb2bb8e2d 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java @@ -21,24 +21,43 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; -@Category({ MapReduceTests.class, SmallTests.class }) +@Category({ MapReduceTests.class, MediumTests.class }) public class TestWALInputFormat { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestWALInputFormat.class); + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.startMiniCluster(); + TEST_UTIL.createWALRootDir(); + } + /** * Test the primitive start/end time filtering. */ @@ -74,4 +93,36 @@ public void testAddFile() { WALInputFormat.addFile(lfss, lfs, now, now); assertEquals(8, lfss.size()); } + + @Test + public void testHandlesArchivedWALFiles() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + JobContext ctx = Mockito.mock(JobContext.class); + Mockito.when(ctx.getConfiguration()).thenReturn(conf); + Job job = Job.getInstance(conf); + TableMapReduceUtil.initCredentialsForCluster(job, conf); + Mockito.when(ctx.getCredentials()).thenReturn(job.getCredentials()); + + // Setup WAL file, then archive it + HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0); + AbstractFSWAL wal = (AbstractFSWAL) rs.getWALs().get(0); + Path walPath = wal.getCurrentFileName(); + TEST_UTIL.getConfiguration().set(FileInputFormat.INPUT_DIR, walPath.toString()); + TEST_UTIL.getConfiguration().set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); + + Path rootDir = CommonFSUtils.getWALRootDir(conf); + Path archiveWal = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + archiveWal = new Path(archiveWal, walPath.getName()); + TEST_UTIL.getTestFileSystem().delete(walPath, true); + TEST_UTIL.getTestFileSystem().mkdirs(archiveWal.getParent()); + TEST_UTIL.getTestFileSystem().create(archiveWal).close(); + + // Test for that we can read from the archived WAL file + WALInputFormat wif = new WALInputFormat(); + List splits = wif.getSplits(ctx); + assertEquals(1, splits.size()); + WALInputFormat.WALSplit split = (WALInputFormat.WALSplit) splits.get(0); + assertEquals(archiveWal.toString(), split.getLogFileName()); + } + } From 78a21032253a1f80733e72577e425cf33c6d3cf4 Mon Sep 17 00:00:00 2001 From: JinHyuk Kim Date: Wed, 30 Jul 2025 10:06:17 +0900 Subject: [PATCH 009/336] HBASE-15625 Make minimum free heap memory percentage configurable (#7179) Signed-off-by: Junegunn Choi --- .../org/apache/hadoop/hbase/HConstants.java | 3 +- .../src/main/resources/hbase-default.xml | 9 ++ .../hadoop/hbase/io/util/MemorySizeUtil.java | 91 ++++++++++++++----- .../hbase/regionserver/HRegionServer.java | 2 +- .../hbase/regionserver/HeapMemoryManager.java | 73 ++++++++------- .../hbase/io/util/TestMemorySizeUtil.java | 89 ++++++++++++++++++ 6 files changed, 208 insertions(+), 59 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 14d7073d5da5..f23f28f8e036 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1089,7 +1089,8 @@ public enum OperationStatusCode { public static final boolean HFILE_PREAD_ALL_BYTES_ENABLED_DEFAULT = false; /* - * Minimum percentage of free heap necessary for a successful cluster startup. + * Default minimum fraction (20%) of free heap required for RegionServer startup, used only when + * 'hbase.regionserver.free.heap.min.memory.size' is not explicitly set. */ public static final float HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD = 0.2f; diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 15b970c7e8d0..7edf00e6b4c1 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -288,6 +288,15 @@ possible configurations would overwhelm and obscure the important. org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter The WAL file writer implementation. + + hbase.regionserver.free.heap.min.memory.size + + Defines the minimum amount of heap memory that must remain free for the RegionServer to start, + specified in bytes or human-readable formats like '512m' for megabytes or '4g' for gigabytes. + If not set, the default is 20% of the total heap size. To disable the check entirely, + set this value to 0. If the combined memory usage of memstore and block cache + exceeds (total heap - this value), the RegionServer will fail to start. + hbase.regionserver.global.memstore.size diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java index 15aeb2153e61..060c10bd63ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java @@ -52,9 +52,14 @@ public class MemorySizeUtil { // Default lower water mark limit is 95% size of memstore size. public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f; + /** + * Configuration key for the absolute amount of heap memory that must remain free for a + * RegionServer to start + */ + public static final String HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY = + "hbase.regionserver.free.heap.min.memory.size"; + private static final Logger LOG = LoggerFactory.getLogger(MemorySizeUtil.class); - // a constant to convert a fraction to a percentage - private static final int CONVERT_TO_PERCENTAGE = 100; private static final String JVM_HEAP_EXCEPTION = "Got an exception while attempting to read " + "information about the JVM heap. Please submit this log information in a bug report and " @@ -78,34 +83,70 @@ public static MemoryUsage safeGetHeapMemoryUsage() { } /** - * Checks whether we have enough heap memory left out after portion for Memstore and Block cache. - * We need atleast 20% of heap left out for other RS functions. + * Validates that heap allocations for MemStore and block cache do not exceed the allowed limit, + * ensuring enough free heap remains for other RegionServer tasks. + * @param conf the configuration to validate + * @throws RuntimeException if the combined allocation exceeds the threshold */ - public static void checkForClusterFreeHeapMemoryLimit(Configuration conf) { + public static void validateRegionServerHeapMemoryAllocation(Configuration conf) { if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) { LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY); } - float globalMemstoreSize = getGlobalMemStoreHeapPercent(conf, false); - int gml = (int) (globalMemstoreSize * CONVERT_TO_PERCENTAGE); - float blockCacheUpperLimit = getBlockCacheHeapPercent(conf); - int bcul = (int) (blockCacheUpperLimit * CONVERT_TO_PERCENTAGE); - if ( - CONVERT_TO_PERCENTAGE - (gml + bcul) - < (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD) - ) { - throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " - + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. Please check " + "the settings for " - + MEMSTORE_SIZE_KEY + " and either " + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " or " - + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " in your configuration. " + MEMSTORE_SIZE_KEY - + "=" + globalMemstoreSize + ", " + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + "=" - + conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY) + ", " - + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + "=" - + conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY) + ". (Note: If both " - + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + " and " - + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " are set, " + "the system will use " - + HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY + ")"); + float memStoreFraction = getGlobalMemStoreHeapPercent(conf, false); + float blockCacheFraction = getBlockCacheHeapPercent(conf); + float minFreeHeapFraction = getRegionServerMinFreeHeapFraction(conf); + + int memStorePercent = (int) (memStoreFraction * 100); + int blockCachePercent = (int) (blockCacheFraction * 100); + int minFreeHeapPercent = (int) (minFreeHeapFraction * 100); + int usedPercent = memStorePercent + blockCachePercent; + int maxAllowedUsed = 100 - minFreeHeapPercent; + + if (usedPercent > maxAllowedUsed) { + throw new RuntimeException(String.format( + "RegionServer heap memory allocation is invalid: total memory usage exceeds 100%% " + + "(memStore + blockCache + requiredFreeHeap). " + + "Check the following configuration values:%n" + " - %s = %.2f%n" + " - %s = %s%n" + + " - %s = %s%n" + " - %s = %s", + MEMSTORE_SIZE_KEY, memStoreFraction, HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY, + conf.get(HConstants.HFILE_BLOCK_CACHE_MEMORY_SIZE_KEY), + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, conf.get(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY), + HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, + conf.get(HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY))); + } + } + + /** + * Retrieve an explicit minimum required free heap size in bytes in the configuration. + * @param conf used to read configs + * @return the minimum required free heap size in bytes, or a negative value if not configured. + * @throws IllegalArgumentException if HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY format is + * invalid + */ + private static long getRegionServerMinFreeHeapInBytes(Configuration conf) { + final String key = HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY; + try { + return Long.parseLong(conf.get(key)); + } catch (NumberFormatException e) { + return (long) StorageSize.getStorageSize(conf.get(key), -1, StorageUnit.BYTES); + } + } + + /** + * Returns the minimum required free heap as a fraction of total heap. + */ + public static float getRegionServerMinFreeHeapFraction(final Configuration conf) { + final MemoryUsage usage = safeGetHeapMemoryUsage(); + if (usage == null) { + return 0; + } + + long minFreeHeapInBytes = getRegionServerMinFreeHeapInBytes(conf); + if (minFreeHeapInBytes >= 0) { + return (float) minFreeHeapInBytes / usage.getMax(); } + + return HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 810e10f1c56d..621cba3775a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -635,7 +635,7 @@ public HRegionServer(final Configuration conf) throws IOException { this.dataFsOk = true; this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); this.eventLoopGroupConfig = setupNetty(this.conf); - MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); HFile.checkHFileVersion(this.conf); checkCodecs(this.conf); this.userProvider = UserProvider.instantiate(conf); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 1b28846efad0..9c4cd5b3ca45 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -47,8 +47,6 @@ public class HeapMemoryManager { private static final Logger LOG = LoggerFactory.getLogger(HeapMemoryManager.class); private static final int CONVERT_TO_PERCENTAGE = 100; - private static final int CLUSTER_MINIMUM_MEMORY_THRESHOLD = - (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD); public static final String BLOCK_CACHE_SIZE_MAX_RANGE_KEY = "hfile.block.cache.size.max.range"; public static final String BLOCK_CACHE_SIZE_MIN_RANGE_KEY = "hfile.block.cache.size.min.range"; @@ -84,6 +82,7 @@ public class HeapMemoryManager { private final boolean tunerOn; private final int defaultChorePeriod; private final float heapOccupancyLowWatermark; + private final float minFreeHeapFraction; private final long maxHeapSize; { @@ -110,6 +109,7 @@ public class HeapMemoryManager { this.memStoreFlusher = memStoreFlusher; this.server = server; this.regionServerAccounting = regionServerAccounting; + this.minFreeHeapFraction = MemorySizeUtil.getRegionServerMinFreeHeapFraction(conf); this.tunerOn = doInit(conf); this.defaultChorePeriod = conf.getInt(HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD); @@ -130,7 +130,7 @@ private boolean doInit(Configuration conf) { boolean tuningEnabled = true; globalMemStorePercent = MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false); blockCachePercent = MemorySizeUtil.getBlockCacheHeapPercent(conf); - MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(conf); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); // Initialize max and min range for memstore heap space globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercent); @@ -184,28 +184,11 @@ private boolean doInit(Configuration conf) { tuningEnabled = false; } - int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE); - int bcul = (int) ((blockCachePercentMinRange) * CONVERT_TO_PERCENTAGE); - if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { - throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " - + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. Please check the settings for " - + MEMSTORE_SIZE_MAX_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY - + " in your configuration. " + MEMSTORE_SIZE_MAX_RANGE_KEY + " is " - + globalMemStorePercentMaxRange + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " is " - + blockCachePercentMinRange); - } - gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE); - bcul = (int) ((blockCachePercentMaxRange) * CONVERT_TO_PERCENTAGE); - if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { - throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds " - + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. Please check the settings for " - + MEMSTORE_SIZE_MIN_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY - + " in your configuration. " + MEMSTORE_SIZE_MIN_RANGE_KEY + " is " - + globalMemStorePercentMinRange + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " is " - + blockCachePercentMaxRange); - } + checkHeapMemoryLimits(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange, + BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange); + checkHeapMemoryLimits(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange, + BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange); + return tuningEnabled; } @@ -241,6 +224,31 @@ public float getHeapOccupancyPercent() { : this.heapOccupancyPercent; } + private boolean isHeapMemoryUsageExceedingLimit(float memStoreFraction, + float blockCacheFraction) { + // Use integer percentage to avoid subtle float precision issues and ensure consistent + // comparison. This also maintains backward compatibility with previous logic relying on int + // truncation. + int memStorePercent = (int) (memStoreFraction * CONVERT_TO_PERCENTAGE); + int blockCachePercent = (int) (blockCacheFraction * CONVERT_TO_PERCENTAGE); + int minFreeHeapPercent = (int) (this.minFreeHeapFraction * CONVERT_TO_PERCENTAGE); + + return memStorePercent + blockCachePercent + minFreeHeapPercent > CONVERT_TO_PERCENTAGE; + } + + private void checkHeapMemoryLimits(String memStoreConfKey, float memStoreFraction, + String blockCacheConfKey, float blockCacheFraction) { + if (isHeapMemoryUsageExceedingLimit(memStoreFraction, blockCacheFraction)) { + throw new RuntimeException(String.format( + "Current heap configuration for MemStore and BlockCache exceeds the allowed heap usage. " + + "At least %.2f of the heap must remain free to ensure stable RegionServer operation. " + + "Please check the settings for %s and %s in your configuration. " + + "%s is %.2f and %s is %.2f", + minFreeHeapFraction, memStoreConfKey, blockCacheConfKey, memStoreConfKey, memStoreFraction, + blockCacheConfKey, blockCacheFraction)); + } + } + private class HeapMemoryTunerChore extends ScheduledChore implements FlushRequestListener { private HeapMemoryTuner heapMemTuner; private AtomicLong blockedFlushCount = new AtomicLong(); @@ -363,14 +371,15 @@ private void tune() { + blockCachePercentMaxRange + ". Resetting blockCacheSize to min size"); blockCacheSize = blockCachePercentMaxRange; } - int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE); - int bcul = (int) ((blockCacheSize) * CONVERT_TO_PERCENTAGE); - if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) { + + if (isHeapMemoryUsageExceedingLimit(memstoreSize, blockCacheSize)) { LOG.info("Current heap configuration from HeapMemoryTuner exceeds " - + "the threshold required for successful cluster operation. " - + "The combined value cannot exceed 0.8. " + MemorySizeUtil.MEMSTORE_SIZE_KEY + " is " - + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is " + blockCacheSize); - // TODO can adjust the value so as not exceed 80%. Is that correct? may be. + + "the allowed heap usage. At least " + minFreeHeapFraction + + " of the heap must remain free to ensure stable RegionServer operation. " + + MemorySizeUtil.MEMSTORE_SIZE_KEY + " is " + memstoreSize + " and " + + HFILE_BLOCK_CACHE_SIZE_KEY + " is " + blockCacheSize); + // NOTE: In the future, we might adjust values to not exceed limits, + // but for now tuning is skipped if over threshold. } else { int memStoreDeltaSize = (int) ((memstoreSize - globalMemStorePercent) * CONVERT_TO_PERCENTAGE); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java new file mode 100644 index 000000000000..5f00c34dbcb0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/util/TestMemorySizeUtil.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MiscTests.class, SmallTests.class }) +public class TestMemorySizeUtil { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMemorySizeUtil.class); + + private Configuration conf; + + @Before + public void setup() { + conf = new Configuration(); + } + + @Test + public void testValidateRegionServerHeapMemoryAllocation() { + // when memstore size + block cache size + default free heap min size == 1.0 + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4f); + conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.4f); + assertEquals(HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD, 0.2f, 0.0f); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); + + // when memstore size + block cache size + default free heap min size > 1.0 + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.5f); + assertThrows(RuntimeException.class, + () -> MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf)); + + // when free heap min size is set to 0, it should not throw an exception + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.5f); + conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f); + conf.setLong(MemorySizeUtil.HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, 0L); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); + + // when free heap min size is set to a negative value, it should be regarded as default value + conf.setLong(MemorySizeUtil.HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, -1024L); + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4f); + conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.4f); + MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf); + + conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.41f); + assertThrows(RuntimeException.class, + () -> MemorySizeUtil.validateRegionServerHeapMemoryAllocation(conf)); + } + + @Test + public void testGetRegionServerMinFreeHeapFraction() { + // when setting is not set, it should return the default value + conf.set(MemorySizeUtil.HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, ""); + float minFreeHeapFraction = MemorySizeUtil.getRegionServerMinFreeHeapFraction(conf); + assertEquals(HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD, minFreeHeapFraction, 0.0f); + + // when setting to 0, it should return 0.0f + conf.set(MemorySizeUtil.HBASE_REGION_SERVER_FREE_HEAP_MIN_MEMORY_SIZE_KEY, "0"); + minFreeHeapFraction = MemorySizeUtil.getRegionServerMinFreeHeapFraction(conf); + assertEquals(0.0f, minFreeHeapFraction, 0.0f); + } +} From 202c2e7d43ac3dcab66d7701647186dbef5a7c6f Mon Sep 17 00:00:00 2001 From: Jaehui Lee Date: Thu, 31 Jul 2025 00:32:01 +0900 Subject: [PATCH 010/336] HBASE-29482 Bulkload fails with viewfs authentication error (#7180) Signed-off-by: Junegunn Choi --- .../org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java index 875556b11d81..a324b01e1c25 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java @@ -428,7 +428,9 @@ private Map performBulkLoad(Admin admin, Table table, LOG.warn("Secure bulk load has been integrated into HBase core."); } - fsDelegationToken.acquireDelegationToken(queue.peek().getFilePath().getFileSystem(getConf())); + Path path = queue.peek().getFilePath(); + FileSystem fs = path.getFileSystem(getConf()).resolvePath(path).getFileSystem(getConf()); + fsDelegationToken.acquireDelegationToken(fs); bulkToken = secureClient.prepareBulkLoad(admin.getConnection()); Pair, Set> pair = null; From 2c92069bda1ce70dc15ec75fcae63657eeaf45c2 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 31 Jul 2025 19:39:36 +0200 Subject: [PATCH 011/336] HBASE-29481 Make TLS protocols and cipher list configurable for HTTPS InfoServer (#7178) Signed-off-by: Nihal Jain (cherry picked from commit daefb0204f497142a64e590d2330f028ed0fe5f7) --- .../apache/hadoop/hbase/http/HttpServer.java | 40 +++++++++++++++++++ .../apache/hadoop/hbase/http/InfoServer.java | 11 ++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 36a101b6ac73..6012b24ec543 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -228,7 +228,10 @@ public static class Builder { private String usernameConfKey; private String keytabConfKey; private boolean needsClientAuth; + private String includeCiphers; private String excludeCiphers; + private String includeProtocols; + private String excludeProtocols; private String hostName; private String appDir = APP_DIR; @@ -401,10 +404,32 @@ public Builder setLogDir(String logDir) { return this; } + @Deprecated + // Use setExcludeCiphers() which supports the fluent builder API public void excludeCiphers(String excludeCiphers) { this.excludeCiphers = excludeCiphers; } + public Builder setExcludeCiphers(String excludeCiphers) { + this.excludeCiphers = excludeCiphers; + return this; + } + + public Builder setIncludeCiphers(String includeCiphers) { + this.includeCiphers = includeCiphers; + return this; + } + + public Builder setIncludeProtocols(String includeProtocols) { + this.includeProtocols = includeProtocols; + return this; + } + + public Builder setExcludeProtocols(String excludeProtocols) { + this.excludeProtocols = excludeProtocols; + return this; + } + public HttpServer build() throws IOException { // Do we still need to assert this non null name if it is deprecated? @@ -466,6 +491,21 @@ public HttpServer build() throws IOException { sslCtxFactory.setTrustStorePassword(trustStorePassword); } + if (includeProtocols != null && !includeProtocols.trim().isEmpty()) { + sslCtxFactory.setIncludeProtocols(StringUtils.getTrimmedStrings(includeProtocols)); + LOG.debug("Included TLS Protocol List:" + includeProtocols); + } + + if (excludeProtocols != null && !excludeProtocols.trim().isEmpty()) { + sslCtxFactory.setExcludeProtocols(StringUtils.getTrimmedStrings(excludeProtocols)); + LOG.debug("Excluded TLS Protocol List:" + excludeProtocols); + } + + if (includeCiphers != null && !includeCiphers.trim().isEmpty()) { + sslCtxFactory.setIncludeCipherSuites(StringUtils.getTrimmedStrings(includeCiphers)); + LOG.debug("Included SSL Cipher List:" + includeCiphers); + } + if (excludeCiphers != null && !excludeCiphers.trim().isEmpty()) { sslCtxFactory.setExcludeCipherSuites(StringUtils.getTrimmedStrings(excludeCiphers)); LOG.debug("Excluded SSL Cipher List:" + excludeCiphers); diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index aa25ef427629..6a08e21df97d 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -67,6 +67,9 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, builder.setLogDir(logDir); } if (httpConfig.isSecure()) { + // We are using the Hadoop HTTP server config properties. + // This makes it easy to keep in sync with Hadoop's UI servers, but hard to set this + // separately for HBase. builder .keyPassword(HBaseConfiguration.getPassword(c, "ssl.server.keystore.keypassword", null)) .keyStore(c.get("ssl.server.keystore.location"), @@ -74,8 +77,12 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, c.get("ssl.server.keystore.type", "jks")) .trustStore(c.get("ssl.server.truststore.location"), HBaseConfiguration.getPassword(c, "ssl.server.truststore.password", null), - c.get("ssl.server.truststore.type", "jks")); - builder.excludeCiphers(c.get("ssl.server.exclude.cipher.list")); + c.get("ssl.server.truststore.type", "jks")) + // The ssl.server.*.protocols properties do not exist in Hadoop at the time of writing. + .setIncludeProtocols(c.get("ssl.server.include.protocols")) + .setExcludeProtocols(c.get("ssl.server.exclude.protocols")) + .setIncludeCiphers(c.get("ssl.server.include.cipher.list")) + .setExcludeCiphers(c.get("ssl.server.exclude.cipher.list")); } final String httpAuthType = c.get(HttpServer.HTTP_UI_AUTHENTICATION, "").toLowerCase(); From 92d4774c07e4b13b1c04f99911bc2d947be3256a Mon Sep 17 00:00:00 2001 From: Dimas Shidqi Parikesit Date: Sat, 2 Aug 2025 10:39:06 -0400 Subject: [PATCH 012/336] HBASE-29296 Missing critical snapshot expiration checks (#6970) Signed-off-by: Peng Lu --- .../impl/IncrementalTableBackupClient.java | 10 + .../hadoop/hbase/backup/util/RestoreTool.java | 14 ++ .../hbase/backup/TestBackupRestoreExpiry.java | 232 ++++++++++++++++++ .../master/procedure/SnapshotProcedure.java | 8 + .../master/snapshot/TakeSnapshotHandler.java | 10 + .../client/TestSnapshotWithTTLFromClient.java | 8 +- .../TestSnapshotProcedureEarlyExpiration.java | 102 ++++++++ .../snapshot/TestTakeSnapshotHandler.java | 9 + 8 files changed, 389 insertions(+), 4 deletions(-) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 1e07c026f0aa..d51f1f471514 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.util.Tool; @@ -63,6 +65,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; /** @@ -541,6 +544,13 @@ private void verifyCfCompatibility(Set tables, SnapshotDescriptionUtils.readSnapshotInfo(fs, manifestDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, manifestDir, snapshotDescription); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(), + snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException( + ProtobufUtil.createSnapshotDesc(snapshotDescription)); + } ColumnFamilyDescriptor[] backupCfs = manifest.getTableDescriptor().getColumnFamilies(); if (!areCfsCompatible(currentCfs, backupCfs)) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java index 7549b9a8c69a..50b47565d743 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -54,6 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; /** @@ -265,6 +267,12 @@ TableDescriptor getTableDesc(TableName tableName) throws IOException { Path tableInfoPath = this.getTableInfoPath(tableName); SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(desc.getTtl(), desc.getCreationTime(), + EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(desc)); + } TableDescriptor tableDescriptor = manifest.getTableDescriptor(); if (!tableDescriptor.getTableName().equals(tableName)) { LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " @@ -310,6 +318,12 @@ private void createAndRestoreTable(Connection conn, TableName tableName, TableNa SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(desc.getTtl(), desc.getCreationTime(), + EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(desc)); + } tableDescriptor = manifest.getTableDescriptor(); } else { tableDescriptor = getTableDesc(tableName); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java new file mode 100644 index 000000000000..4126ed748951 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.LogRoller; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +@Category(LargeTests.class) +public class TestBackupRestoreExpiry extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBackupRestoreExpiry.class); + + @BeforeClass + public static void setUp() throws Exception { + TEST_UTIL = new HBaseTestingUtil(); + conf1 = TEST_UTIL.getConfiguration(); + conf1.setLong(HConstants.DEFAULT_SNAPSHOT_TTL_CONFIG_KEY, 30); + autoRestoreOnFailure = true; + useSecondCluster = false; + setUpHelper(); + } + + public void ensurePreviousBackupTestsAreCleanedUp() throws Exception { + TEST_UTIL.flush(table1); + TEST_UTIL.flush(table2); + + TEST_UTIL.truncateTable(table1).close(); + TEST_UTIL.truncateTable(table2).close(); + + if (TEST_UTIL.getAdmin().tableExists(table1_restore)) { + TEST_UTIL.flush(table1_restore); + TEST_UTIL.truncateTable(table1_restore).close(); + } + + TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(rst -> { + try { + LogRoller walRoller = rst.getRegionServer().getWalRoller(); + walRoller.requestRollAll(); + walRoller.waitUntilWalRollFinished(); + } catch (Exception ignored) { + } + }); + + try (Table table = TEST_UTIL.getConnection().getTable(table1)) { + loadTable(table); + } + + try (Table table = TEST_UTIL.getConnection().getTable(table2)) { + loadTable(table); + } + } + + @Test + public void testSequentially() throws Exception { + try { + testRestoreOnExpiredFullBackup(); + } catch (Exception e) { + throw e; + } finally { + ensurePreviousBackupTestsAreCleanedUp(); + } + + try { + testIncrementalBackupOnExpiredFullBackup(); + } catch (Exception e) { + throw e; + } finally { + ensurePreviousBackupTestsAreCleanedUp(); + } + } + + public void testRestoreOnExpiredFullBackup() throws Exception { + byte[] mobFam = Bytes.toBytes("mob"); + + List tables = Lists.newArrayList(table1); + TableDescriptor newTable1Desc = + TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + Connection conn = TEST_UTIL.getConnection(); + BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String fullBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(fullBackupId)); + + TableName[] fromTables = new TableName[] { table1 }; + TableName[] toTables = new TableName[] { table1_restore }; + + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time + 30s + @Override + public long currentTime() { + return System.currentTimeMillis() + (30 * 1000); + } + }); + + assertThrows(SnapshotTTLExpiredException.class, () -> { + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, + fromTables, toTables, true, true)); + }); + + EnvironmentEdgeManager.reset(); + backupAdmin.close(); + } + + public void testIncrementalBackupOnExpiredFullBackup() throws Exception { + byte[] mobFam = Bytes.toBytes("mob"); + + List tables = Lists.newArrayList(table1); + TableDescriptor newTable1Desc = + TableDescriptorBuilder.newBuilder(table1Desc).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(mobFam).setMobEnabled(true).setMobThreshold(5L).build()).build(); + TEST_UTIL.getAdmin().modifyTable(newTable1Desc); + + Connection conn = TEST_UTIL.getConnection(); + BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); + BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); + String fullBackupId = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(fullBackupId)); + + TableName[] fromTables = new TableName[] { table1 }; + TableName[] toTables = new TableName[] { table1_restore }; + + List preRestoreBackupFiles = getBackupFiles(); + backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, fullBackupId, false, + fromTables, toTables, true, true)); + List postRestoreBackupFiles = getBackupFiles(); + + // Check that the backup files are the same before and after the restore process + Assert.assertEquals(postRestoreBackupFiles, preRestoreBackupFiles); + Assert.assertEquals(TEST_UTIL.countRows(table1_restore), NB_ROWS_IN_BATCH); + + int ROWS_TO_ADD = 1_000; + // different IDs so that rows don't overlap + insertIntoTable(conn, table1, famName, 3, ROWS_TO_ADD); + insertIntoTable(conn, table1, mobFam, 4, ROWS_TO_ADD); + + Admin admin = conn.getAdmin(); + List currentRegions = TEST_UTIL.getHBaseCluster().getRegions(table1); + for (HRegion region : currentRegions) { + byte[] name = region.getRegionInfo().getEncodedNameAsBytes(); + admin.splitRegionAsync(name).get(); + } + + TEST_UTIL.waitTableAvailable(table1); + + // Make sure we've split regions + assertNotEquals(currentRegions, TEST_UTIL.getHBaseCluster().getRegions(table1)); + + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time + 30s + @Override + public long currentTime() { + return System.currentTimeMillis() + (30 * 1000); + } + }); + + IOException e = assertThrows(IOException.class, () -> { + backupAdmin + .backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)); + }); + assertTrue(e.getCause() instanceof SnapshotTTLExpiredException); + + EnvironmentEdgeManager.reset(); + backupAdmin.close(); + } + + private List getBackupFiles() throws IOException { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + RemoteIterator iter = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); + List files = new ArrayList<>(); + + while (iter.hasNext()) { + files.add(iter.next()); + } + + return files; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java index 572cdce0cf6f..17a9c083896a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java @@ -50,7 +50,9 @@ import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ModifyRegionUtils; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.yetus.audience.InterfaceAudience; @@ -159,6 +161,12 @@ protected Flow executeFromState(MasterProcedureEnv env, SnapshotState state) if (isSnapshotCorrupted()) { throw new CorruptedSnapshotException(snapshot.getName()); } + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(snapshot.getTtl(), + snapshot.getCreationTime(), EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(snapshot)); + } completeSnapshot(env); setNextState(SnapshotState.SNAPSHOT_POST_OPERATION); return Flow.HAS_MORE_STATE; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java index b24f79494045..a3a25e3d3e3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java @@ -48,7 +48,9 @@ import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.KeeperException; @@ -225,6 +227,14 @@ public void process() { status.setStatus("Verifying snapshot: " + snapshot.getName()); verifier.verifySnapshot(workingDir, true); + // HBASE-29296 check snapshot is not expired + if ( + SnapshotDescriptionUtils.isExpiredSnapshot(snapshot.getTtl(), snapshot.getCreationTime(), + EnvironmentEdgeManager.currentTime()) + ) { + throw new SnapshotTTLExpiredException(ProtobufUtil.createSnapshotDesc(snapshot)); + } + // complete the snapshot, atomically moving from tmp to .snapshot dir. SnapshotDescriptionUtils.completeSnapshot(this.snapshotDir, this.workingDir, this.rootFs, this.workingDirFs, this.conf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithTTLFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithTTLFromClient.java index 4309b922b8fd..9713569e4068 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithTTLFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithTTLFromClient.java @@ -143,7 +143,7 @@ public void testRestoreSnapshotFailsDueToTTLExpired() throws Exception { assertTrue(UTIL.getAdmin().tableExists(TABLE_NAME)); // create snapshot fo given table with specified ttl - createSnapshotWithTTL(TABLE_NAME, snapshotName, 1); + createSnapshotWithTTL(TABLE_NAME, snapshotName, 5); Admin admin = UTIL.getAdmin(); // Disable and drop table @@ -152,7 +152,7 @@ public void testRestoreSnapshotFailsDueToTTLExpired() throws Exception { assertFalse(UTIL.getAdmin().tableExists(TABLE_NAME)); // Sleep so that TTL may expire - Threads.sleep(2000); + Threads.sleep(10000); // restore snapshot which has expired try { @@ -192,13 +192,13 @@ public void testCloneSnapshotFailsDueToTTLExpired() throws Exception { assertTrue(UTIL.getAdmin().tableExists(TABLE_NAME)); // create snapshot fo given table with specified ttl - createSnapshotWithTTL(TABLE_NAME, snapshotName, 1); + createSnapshotWithTTL(TABLE_NAME, snapshotName, 5); Admin admin = UTIL.getAdmin(); assertTrue(UTIL.getAdmin().tableExists(TABLE_NAME)); // Sleep so that TTL may expire - Threads.sleep(2000); + Threads.sleep(10000); // clone snapshot which has expired try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java new file mode 100644 index 000000000000..0870f16face1 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.RegionSplitter; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; + +public class TestSnapshotProcedureEarlyExpiration extends TestSnapshotProcedure { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSnapshotProcedureEarlyExpiration.class); + + @Before + @Override + public void setup() throws Exception { // Copied from TestSnapshotProcedure with modified + // SnapshotDescription + TEST_UTIL = new HBaseTestingUtil(); + Configuration config = TEST_UTIL.getConfiguration(); + // using SnapshotVerifyProcedure to verify snapshot + config.setInt("hbase.snapshot.remote.verify.threshold", 1); + // disable info server. Info server is useful when we run unit tests locally, + // but it will + // fails integration testing of jenkins. + // config.setInt(HConstants.MASTER_INFO_PORT, 8080); + + // delay dispatch so that we can do something, for example kill a target server + config.setInt(RemoteProcedureDispatcher.DISPATCH_DELAY_CONF_KEY, 10000); + config.setInt(RemoteProcedureDispatcher.DISPATCH_MAX_QUEUE_SIZE_CONF_KEY, 128); + TEST_UTIL.startMiniCluster(3); + master = TEST_UTIL.getHBaseCluster().getMaster(); + TABLE_NAME = TableName.valueOf(Bytes.toBytes("SPTestTable")); + CF = Bytes.toBytes("cf"); + SNAPSHOT_NAME = "SnapshotProcedureTest"; + + Map properties = new HashMap<>(); + properties.put("TTL", 1L); + snapshot = new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.FLUSH, null, -1, -1, + properties); + + snapshotProto = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); + snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto, master.getConfiguration()); + final byte[][] splitKeys = new RegionSplitter.HexStringSplit().split(10); + Table table = TEST_UTIL.createTable(TABLE_NAME, CF, splitKeys); + TEST_UTIL.loadTable(table, CF, false); + } + + @Test + public void testSnapshotEarlyExpiration() throws Exception { + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + MasterProcedureEnv env = procExec.getEnvironment(); + SnapshotProcedure sp = new SnapshotProcedure(env, snapshotProto); + SnapshotProcedure spySp = getDelayedOnSpecificStateSnapshotProcedure(sp, + procExec.getEnvironment(), SnapshotState.SNAPSHOT_COMPLETE_SNAPSHOT); + + long procId = procExec.submitProcedure(spySp); + + ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), procId); + assertTrue(spySp.isFailed()); + List snapshots = + master.getSnapshotManager().getCompletedSnapshots(); + assertEquals(0, snapshots.size()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestTakeSnapshotHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestTakeSnapshotHandler.java index e9d3b9784d66..d18ef5728d22 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestTakeSnapshotHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestTakeSnapshotHandler.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; @@ -100,6 +101,14 @@ public void testPreparePreserveMaxFileSizeDisabled() throws Exception { assertEquals(-1, UTIL.getAdmin().getDescriptor(cloned).getMaxFileSize()); } + @Test(expected = SnapshotTTLExpiredException.class) + public void testSnapshotEarlyExpiration() throws Exception { + UTIL.startMiniCluster(); + Map snapshotProps = new HashMap<>(); + snapshotProps.put("TTL", 1L); + createTableInsertDataAndTakeSnapshot(snapshotProps); + } + @After public void shutdown() throws Exception { UTIL.shutdownMiniCluster(); From da55aca593520de15be9dd9ceae52eb0f591b9bb Mon Sep 17 00:00:00 2001 From: Peng Lu Date: Tue, 5 Aug 2025 00:08:21 +0800 Subject: [PATCH 013/336] HBASE-29296 Addendum, Missing critical snapshot expiration checks (#7189) Signed-off-by: Nihal Jain --- .../apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java | 4 ++-- .../procedure/TestSnapshotProcedureEarlyExpiration.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java index 4126ed748951..2ef9d237efa7 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreExpiry.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; @@ -64,7 +64,7 @@ public class TestBackupRestoreExpiry extends TestBackupBase { @BeforeClass public static void setUp() throws Exception { - TEST_UTIL = new HBaseTestingUtil(); + TEST_UTIL = new HBaseTestingUtility(); conf1 = TEST_UTIL.getConfiguration(); conf1.setLong(HConstants.DEFAULT_SNAPSHOT_TTL_CONFIG_KEY, 30); autoRestoreOnFailure = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java index 0870f16face1..e5d21159e506 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.client.SnapshotType; @@ -53,7 +53,7 @@ public class TestSnapshotProcedureEarlyExpiration extends TestSnapshotProcedure @Override public void setup() throws Exception { // Copied from TestSnapshotProcedure with modified // SnapshotDescription - TEST_UTIL = new HBaseTestingUtil(); + TEST_UTIL = new HBaseTestingUtility(); Configuration config = TEST_UTIL.getConfiguration(); // using SnapshotVerifyProcedure to verify snapshot config.setInt("hbase.snapshot.remote.verify.threshold", 1); From 9e61771a353320d2354150032b729392f89d5072 Mon Sep 17 00:00:00 2001 From: Prathyush <48905104+prathyush17@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:34:29 +0530 Subject: [PATCH 014/336] HBASE-29477 Add configuration support for custom OutputCommitter in TableOutputFormat (#7185) Signed-off-by: Viraj Jasani Signed-off-by: Pankaj Kumar Reviewed-by: Ujjawal Reviewed-by: Kevin Geiszler (cherry picked from commit c26953584d8fd316e5f1385a4b82c7d645cff8a1) --- .../hbase/mapreduce/TableOutputFormat.java | 24 ++++++++++- .../mapreduce/TestTableOutputFormat.java | 41 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java index 109aeef4ce0c..54a357f5bef2 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; @@ -72,6 +73,16 @@ public class TableOutputFormat extends OutputFormat implemen */ public static final String OUTPUT_CONF_PREFIX = "hbase.mapred.output."; + /** + * The configuration key for specifying a custom + * {@link org.apache.hadoop.mapreduce.OutputCommitter} implementation to be used by + * {@link TableOutputFormat}. The value for this property should be the fully qualified class name + * of the custom committer. If this property is not set, {@link TableOutputCommitter} will be used + * by default. + */ + public static final String OUTPUT_COMMITTER_CLASS = + "hbase.mapreduce.tableoutputformat.output.committer.class"; + /** * Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying * between hbase clusters (the source is picked up from hbase-site.xml). @@ -216,7 +227,18 @@ public void checkOutputSpecs(JobContext context) throws IOException, Interrupted @Override public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException { - return new TableOutputCommitter(); + Configuration hConf = getConf(); + if (hConf == null) { + hConf = context.getConfiguration(); + } + + try { + Class outputCommitter = + hConf.getClass(OUTPUT_COMMITTER_CLASS, TableOutputCommitter.class, OutputCommitter.class); + return ReflectionUtils.newInstance(outputCommitter); + } catch (Exception e) { + throw new IOException("Could not create the configured OutputCommitter", e); + } } @Override diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java index 801099819b76..9170b26564de 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableOutputFormat.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.junit.After; @@ -127,4 +129,43 @@ public void testTableOutputFormatWhenWalIsOFFForDelete() Assert.assertEquals("Durability of the mutation should be SKIP_WAL", Durability.SKIP_WAL, delete.getDurability()); } + + @Test + public void testOutputCommitterConfiguration() throws IOException, InterruptedException { + // 1. Verify it returns the default committer when the property is not set. + conf.unset(TableOutputFormat.OUTPUT_COMMITTER_CLASS); + tableOutputFormat.setConf(conf); + Assert.assertEquals("Should use default committer", TableOutputCommitter.class, + tableOutputFormat.getOutputCommitter(context).getClass()); + + // 2. Verify it returns the custom committer when the property is set. + conf.set(TableOutputFormat.OUTPUT_COMMITTER_CLASS, DummyCommitter.class.getName()); + tableOutputFormat.setConf(conf); + Assert.assertEquals("Should use custom committer", DummyCommitter.class, + tableOutputFormat.getOutputCommitter(context).getClass()); + } + + // Simple dummy committer for testing + public static class DummyCommitter extends OutputCommitter { + @Override + public void setupJob(JobContext jobContext) { + } + + @Override + public void setupTask(TaskAttemptContext taskContext) { + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) { + } + + @Override + public void abortTask(TaskAttemptContext taskContext) { + } + } } From 41c191d3e06609bf563d5b0c1055a24e56c8ac1b Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 5 Aug 2025 13:54:57 -0700 Subject: [PATCH 015/336] HBASE-28919 Soft drop for destructive table actions (branch-2/ branch-2.6) (#7184) Although HFiles are copied to the archive in a destructive schema change, recovery scenarios are not automatic and involve some operator labor to reconstruct the table and re-import the archived data. We can easily prevent the deletion of the HFiles of a deleted table or column family by taking a snapshot of the table immediately prior to any destructive schema actions. We also set a TTL on the snapshot so housekeeping of unwanted HFiles remains no touch. Because we take a table snapshot all table structure and metadata is also captured and saved so fast recovery is possible, as either a restore from snapshot, or a clone from snapshot to a new table. Existing site configuration property prerequisites: * hbase.snapshot.enabled = true ( default is true ) New site configuration properties: * hbase.snapshot.before.destructive.action.enabled = true ( default is false ) * hbase.snapshot.before.destructive.action.ttl = , in seconds ( default 86400 (one day) ) Signed-off-by: Viraj Jasani --- .../org/apache/hadoop/hbase/HConstants.java | 12 + .../src/main/protobuf/MasterProcedure.proto | 13 + .../procedure/DeleteTableProcedure.java | 54 ++++- .../procedure/ModifyTableProcedure.java | 61 ++++- .../procedure/RecoverySnapshotUtils.java | 206 ++++++++++++++++ .../procedure/TruncateRegionProcedure.java | 98 ++++++-- .../procedure/TruncateTableProcedure.java | 52 +++- .../TestDeleteTableProcedureWithRecovery.java | 159 +++++++++++++ .../TestModifyTableProcedureWithRecovery.java | 177 ++++++++++++++ .../procedure/TestRecoverySnapshotUtils.java | 96 ++++++++ ...stTruncateRegionProcedureWithRecovery.java | 224 ++++++++++++++++++ ...estTruncateTableProcedureWithRecovery.java | 166 +++++++++++++ 12 files changed, 1273 insertions(+), 45 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverySnapshotUtils.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedureWithRecovery.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedureWithRecovery.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRecoverySnapshotUtils.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedureWithRecovery.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedureWithRecovery.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index f23f28f8e036..c677e610c721 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1620,6 +1620,18 @@ public enum OperationStatusCode { // User defined Default TTL config key public static final String DEFAULT_SNAPSHOT_TTL_CONFIG_KEY = "hbase.master.snapshot.ttl"; + // Soft drop for destructive table actions configuration + public static final String SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY = + "hbase.snapshot.before.destructive.action.enabled"; + public static final boolean DEFAULT_SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED = false; + + public static final String SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL_KEY = + "hbase.snapshot.before.destructive.action.ttl"; + public static final long DEFAULT_SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL = 86400; // 1 day + + // Table-level attribute name for recovery snapshot TTL override + public static final String TABLE_RECOVERY_SNAPSHOT_TTL_KEY = "RECOVERY_SNAPSHOT_TTL"; + // Regions Recovery based on high storeFileRefCount threshold value public static final String STORE_FILE_REF_COUNT_THRESHOLD = "hbase.regions.recovery.store.file.ref.count"; diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 024cb7b8b002..7c7e3e503bb8 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -77,6 +77,7 @@ enum ModifyTableState { MODIFY_TABLE_CLOSE_EXCESS_REPLICAS = 8; MODIFY_TABLE_ASSIGN_NEW_REPLICAS = 9; MODIFY_TABLE_SYNC_ERASURE_CODING_POLICY = 10; + MODIFY_TABLE_SNAPSHOT = 11; } message ModifyTableStateData { @@ -86,6 +87,7 @@ message ModifyTableStateData { required bool delete_column_family_in_modify = 4; optional bool should_check_descriptor = 5; optional bool reopen_regions = 6; + optional string snapshot_name = 7; } enum TruncateTableState { @@ -96,6 +98,7 @@ enum TruncateTableState { TRUNCATE_TABLE_ADD_TO_META = 5; TRUNCATE_TABLE_ASSIGN_REGIONS = 6; TRUNCATE_TABLE_POST_OPERATION = 7; + TRUNCATE_TABLE_SNAPSHOT = 8; } message TruncateTableStateData { @@ -104,6 +107,7 @@ message TruncateTableStateData { optional TableName table_name = 3; optional TableSchema table_schema = 4; repeated RegionInfo region_info = 5; + optional string snapshot_name = 6; } enum TruncateRegionState { @@ -112,6 +116,13 @@ enum TruncateRegionState { TRUNCATE_REGION_REMOVE = 3; TRUNCATE_REGION_MAKE_ONLINE = 4; TRUNCATE_REGION_POST_OPERATION = 5; + TRUNCATE_REGION_SNAPSHOT = 6; +} + +message TruncateRegionStateData { + required UserInformation user_info = 1; + required RegionInfo region_info = 2; + optional string snapshot_name = 3; } enum DeleteTableState { @@ -121,12 +132,14 @@ enum DeleteTableState { DELETE_TABLE_UPDATE_DESC_CACHE = 4; DELETE_TABLE_UNASSIGN_REGIONS = 5; DELETE_TABLE_POST_OPERATION = 6; + DELETE_TABLE_SNAPSHOT = 7; } message DeleteTableStateData { required UserInformation user_info = 1; required TableName table_name = 2; repeated RegionInfo region_info = 3; + optional string snapshot_name = 4; } enum CreateNamespaceState { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java index 544bb69b79fe..f85d6584b540 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java @@ -67,6 +67,7 @@ public class DeleteTableProcedure extends AbstractStateMachineTableProcedure regions; private TableName tableName; private RetryCounter retryCounter; + private String recoverySnapshotName; public DeleteTableProcedure() { // Required by the Procedure framework to create the procedure on replay @@ -110,6 +111,23 @@ protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState s // Call coprocessors preDelete(env); + // Check if we should create a recover snapshot + if (RecoverySnapshotUtils.isRecoveryEnabled(env)) { + setNextState(DeleteTableState.DELETE_TABLE_SNAPSHOT); + } else { + setNextState(DeleteTableState.DELETE_TABLE_CLEAR_FS_LAYOUT); + } + break; + case DELETE_TABLE_SNAPSHOT: + // Create recovery snapshot procedure as child procedure + recoverySnapshotName = RecoverySnapshotUtils.generateSnapshotName(getTableName()); + SnapshotProcedure snapshotProcedure = + RecoverySnapshotUtils.createSnapshotProcedure(env, getTableName(), recoverySnapshotName, + env.getMasterServices().getTableDescriptors().get(tableName)); + // Submit snapshot procedure as child procedure + addChildProcedure(snapshotProcedure); + LOG.debug("Creating recovery snapshot {} for table {} before deletion", + recoverySnapshotName, getTableName()); setNextState(DeleteTableState.DELETE_TABLE_CLEAR_FS_LAYOUT); break; case DELETE_TABLE_CLEAR_FS_LAYOUT: @@ -171,22 +189,34 @@ protected boolean abort(MasterProcedureEnv env) { @Override protected void rollbackState(final MasterProcedureEnv env, final DeleteTableState state) { - if (state == DeleteTableState.DELETE_TABLE_PRE_OPERATION) { - // nothing to rollback, pre-delete is just table-state checks. - // We can fail if the table does not exist or is not disabled. - // TODO: coprocessor rollback semantic is still undefined. - releaseSyncLatch(); - return; + switch (state) { + case DELETE_TABLE_PRE_OPERATION: + // nothing to rollback, pre-delete is just table-state checks. + // We can fail if the table does not exist or is not disabled. + // TODO: coprocessor rollback semantic is still undefined. + releaseSyncLatch(); + return; + case DELETE_TABLE_SNAPSHOT: + // Handle recovery snapshot rollback. There is no DeleteSnapshotProcedure as such to use + // here directly as a child procedure, so we call a utility method to delete the snapshot + // which uses the SnapshotManager to delete the snapshot. + if (recoverySnapshotName != null) { + RecoverySnapshotUtils.deleteRecoverySnapshot(env, recoverySnapshotName, getTableName()); + recoverySnapshotName = null; + } + return; + default: + // Delete from other states doesn't have a rollback. The execution will succeed, at some + // point. + throw new UnsupportedOperationException("unhandled state=" + state); } - - // The delete doesn't have a rollback. The execution will succeed, at some point. - throw new UnsupportedOperationException("unhandled state=" + state); } @Override protected boolean isRollbackSupported(final DeleteTableState state) { switch (state) { case DELETE_TABLE_PRE_OPERATION: + case DELETE_TABLE_SNAPSHOT: return true; default: return false; @@ -236,6 +266,9 @@ protected void serializeStateData(ProcedureStateSerializer serializer) throws IO state.addRegionInfo(ProtobufUtil.toRegionInfo(hri)); } } + if (recoverySnapshotName != null) { + state.setSnapshotName(recoverySnapshotName); + } serializer.serialize(state.build()); } @@ -255,6 +288,9 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws regions.add(ProtobufUtil.toRegionInfo(hri)); } } + if (state.hasSnapshotName()) { + recoverySnapshotName = state.getSnapshotName(); + } } private boolean prepareDelete(final MasterProcedureEnv env) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index d35a49169a57..5b51a5662db9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -61,6 +61,8 @@ public class ModifyTableProcedure extends AbstractStateMachineTableProcedure + * The naming convention is: auto_{table}_{timestamp} + * @param tableName the table name + * @return the generated snapshot name + */ + public static String generateSnapshotName(final TableName tableName) { + return generateSnapshotName(tableName, EnvironmentEdgeManager.currentTime()); + } + + /** + * Generates a recovery snapshot name. + *

+ * The naming convention is: auto_{table}_{timestamp} + * @param tableName the table name + * @param timestamp the timestamp when the snapshot was initiated + * @return the generated snapshot name + */ + public static String generateSnapshotName(final TableName tableName, final long timestamp) { + return "auto_" + tableName.getNameAsString() + "_" + timestamp; + } + + /** + * Creates a SnapshotDescription for the recovery snapshot for a given operation. + * @param tableName the table name + * @param snapshotName the snapshot name + * @return SnapshotDescription for the recovery snapshot + */ + public static SnapshotProtos.SnapshotDescription + buildSnapshotDescription(final TableName tableName, final String snapshotName) { + return buildSnapshotDescription(tableName, snapshotName, 0, + SnapshotProtos.SnapshotDescription.Type.FLUSH); + } + + /** + * Creates a SnapshotDescription for the recovery snapshot for a given operation. + * @param tableName the table name + * @param snapshotName the snapshot name + * @param ttl the TTL for the snapshot in seconds (0 means no TTL) + * @param type the type of snapshot to create + * @return SnapshotDescription for the recovery snapshot + */ + public static SnapshotProtos.SnapshotDescription buildSnapshotDescription( + final TableName tableName, final String snapshotName, final long ttl, + final SnapshotProtos.SnapshotDescription.Type type) { + SnapshotProtos.SnapshotDescription.Builder builder = + SnapshotProtos.SnapshotDescription.newBuilder(); + builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION); + builder.setName(snapshotName); + builder.setTable(tableName.getNameAsString()); + builder.setType(type); + builder.setCreationTime(EnvironmentEdgeManager.currentTime()); + builder.setTtl(ttl); + return builder.build(); + } + + /** + * Creates a SnapshotProcedure for soft drop functionality. + *

+ * This method should be called from procedures that need to create a snapshot before performing + * destructive operations. It will check for table-level TTL overrides. + * @param env MasterProcedureEnv + * @param tableName the table name + * @param snapshotName the name for the snapshot + * @param tableDescriptor the table descriptor to check for table-level TTL override + * @return SnapshotProcedure that can be added as a child procedure + * @throws IOException if snapshot creation fails + */ + public static SnapshotProcedure createSnapshotProcedure(final MasterProcedureEnv env, + final TableName tableName, final String snapshotName, final TableDescriptor tableDescriptor) + throws IOException { + return new SnapshotProcedure(env, + buildSnapshotDescription(tableName, snapshotName, + getRecoverySnapshotTtl(env, tableDescriptor), + env.getMasterServices().getTableStateManager().isTableState(tableName, + org.apache.hadoop.hbase.client.TableState.State.DISABLED) + ? SnapshotProtos.SnapshotDescription.Type.SKIPFLUSH + : SnapshotProtos.SnapshotDescription.Type.FLUSH)); + } + + /** + * Deletes a recovery snapshot during rollback scenarios. + *

+ * This method should be called during procedure rollback to clean up any snapshots that were + * created before the failure. + * @param env MasterProcedureEnv + * @param snapshotName the name of the snapshot to delete + * @param tableName the table name (for logging) + */ + public static void deleteRecoverySnapshot(final MasterProcedureEnv env, final String snapshotName, + final TableName tableName) { + try { + LOG.debug("Deleting recovery snapshot {} for table {} during rollback", snapshotName, + tableName); + SnapshotManager snapshotManager = env.getMasterServices().getSnapshotManager(); + if (snapshotManager == null) { + LOG.warn("SnapshotManager is not available, cannot delete recovery snapshot {}", + snapshotName); + return; + } + // Delete the snapshot using the snapshot manager. The SnapshotManager will handle existence + // checks. + snapshotManager.deleteSnapshot(buildSnapshotDescription(tableName, snapshotName)); + LOG.info("Successfully deleted recovery snapshot {} for table {} during rollback", + snapshotName, tableName); + } catch (SnapshotDoesNotExistException e) { + // Expected during rollback if the snapshot was never created or already cleaned up. + LOG.debug("Recovery snapshot {} for table {} does not exist, skipping", snapshotName, + tableName); + } catch (Exception e) { + // During rollback, we don't want to fail the rollback process due to snapshot cleanup + // issues. Log the error and continue. The snapshot can be manually cleaned up later. + LOG.warn("Failed to delete recovery snapshot {} for table {} during rollback: {}. " + + "Manual cleanup may be required.", snapshotName, tableName, e.getMessage()); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java index 5e907c1681ac..993aca6dd435 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java @@ -26,19 +26,24 @@ import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.assignment.RegionStateNode; import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateRegionState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateRegionStateData; @InterfaceAudience.Private public class TruncateRegionProcedure extends AbstractStateMachineRegionProcedure { private static final Logger LOG = LoggerFactory.getLogger(TruncateRegionProcedure.class); + private String recoverySnapshotName; + @SuppressWarnings("unused") public TruncateRegionProcedure() { // Required by the Procedure framework to create the procedure on replay @@ -75,6 +80,24 @@ assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed() : "Can't truncate replicas directly. " + "Replicas are auto-truncated when their primary is truncated."; preTruncate(env); + + // Check if we should create a recovery snapshot + if (RecoverySnapshotUtils.isRecoveryEnabled(env)) { + setNextState(TruncateRegionState.TRUNCATE_REGION_SNAPSHOT); + } else { + setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE); + } + break; + case TRUNCATE_REGION_SNAPSHOT: + // Create recovery snapshot procedure as child procedure + recoverySnapshotName = RecoverySnapshotUtils.generateSnapshotName(getTableName()); + SnapshotProcedure snapshotProcedure = + RecoverySnapshotUtils.createSnapshotProcedure(env, getTableName(), recoverySnapshotName, + env.getMasterServices().getTableDescriptors().get(getTableName())); + // Submit snapshot procedure as child procedure + addChildProcedure(snapshotProcedure); + LOG.debug("Creating recovery snapshot {} for table {} before truncating region {}", + recoverySnapshotName, getTableName(), getRegion().getRegionNameAsString()); setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE); break; case TRUNCATE_REGION_MAKE_OFFLINE: @@ -124,22 +147,32 @@ private void deleteRegionFromFileSystem(final MasterProcedureEnv env) throws IOE @Override protected void rollbackState(final MasterProcedureEnv env, final TruncateRegionState state) throws IOException { - if (state == TruncateRegionState.TRUNCATE_REGION_PRE_OPERATION) { - // Nothing to rollback, pre-truncate is just table-state checks. - return; - } - if (state == TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE) { - RegionStateNode regionNode = - env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); - if (regionNode == null) { - // Region was unassigned by state TRUNCATE_REGION_MAKE_OFFLINE. - // So Assign it back - addChildProcedure(createAssignProcedures(env)); - } - return; + switch (state) { + case TRUNCATE_REGION_PRE_OPERATION: + // Nothing to rollback, pre-truncate is just table-state checks. + return; + case TRUNCATE_REGION_SNAPSHOT: + // Handle recovery snapshot rollback. There is no DeleteSnapshotProcedure as such to use + // here directly as a child procedure, so we call a utility method to delete the snapshot + // which uses the SnapshotManager to delete the snapshot. + if (recoverySnapshotName != null) { + RecoverySnapshotUtils.deleteRecoverySnapshot(env, recoverySnapshotName, getTableName()); + recoverySnapshotName = null; + } + return; + case TRUNCATE_REGION_MAKE_OFFLINE: + RegionStateNode regionNode = + env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); + if (regionNode == null) { + // Region was unassigned by state TRUNCATE_REGION_MAKE_OFFLINE. + // So Assign it back + addChildProcedure(createAssignProcedures(env)); + } + return; + default: + // The truncate doesn't have a rollback. The execution will succeed, at some point. + throw new UnsupportedOperationException("unhandled state=" + state); } - // The truncate doesn't have a rollback. The execution will succeed, at some point. - throw new UnsupportedOperationException("unhandled state=" + state); } @Override @@ -151,7 +184,7 @@ protected void completionCleanup(final MasterProcedureEnv env) { protected boolean isRollbackSupported(final TruncateRegionState state) { switch (state) { case TRUNCATE_REGION_PRE_OPERATION: - return true; + case TRUNCATE_REGION_SNAPSHOT: case TRUNCATE_REGION_MAKE_OFFLINE: return true; default: @@ -208,6 +241,29 @@ public TableOperationType getTableOperationType() { return TableOperationType.REGION_TRUNCATE; } + @Override + protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException { + super.serializeStateData(serializer); + TruncateRegionStateData.Builder state = TruncateRegionStateData.newBuilder() + .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) + .setRegionInfo(ProtobufUtil.toRegionInfo(getRegion())); + if (recoverySnapshotName != null) { + state.setSnapshotName(recoverySnapshotName); + } + serializer.serialize(state.build()); + } + + @Override + protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException { + super.deserializeStateData(serializer); + TruncateRegionStateData state = serializer.deserialize(TruncateRegionStateData.class); + setUser(MasterProcedureUtil.toUserInfo(state.getUserInfo())); + setRegion(ProtobufUtil.toRegionInfo(state.getRegionInfo())); + if (state.hasSnapshotName()) { + recoverySnapshotName = state.getSnapshotName(); + } + } + private TransitRegionStateProcedure createUnAssignProcedures(MasterProcedureEnv env) throws IOException { return env.getAssignmentManager().createOneUnassignProcedure(getRegion(), true); @@ -216,4 +272,14 @@ private TransitRegionStateProcedure createUnAssignProcedures(MasterProcedureEnv private TransitRegionStateProcedure createAssignProcedures(MasterProcedureEnv env) { return env.getAssignmentManager().createOneAssignProcedure(getRegion(), true); } + + @Override + protected boolean holdLock(MasterProcedureEnv env) { + if (RecoverySnapshotUtils.isRecoveryEnabled(env)) { + // If we are to take a recovery snapshot before deleting the region we will need to allow the + // snapshot procedure to lock the table. + return false; + } + return super.holdLock(env); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java index 78d8fdf3bbcb..028eff1821a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java @@ -49,6 +49,7 @@ public class TruncateTableProcedure extends AbstractStateMachineTableProcedure regions; private TableDescriptor tableDescriptor; private TableName tableName; + private String recoverySnapshotName; public TruncateTableProcedure() { // Required by the Procedure framework to create the procedure on replay @@ -97,6 +98,23 @@ protected Flow executeFromState(final MasterProcedureEnv env, TruncateTableState // the procedure stage and can get recovered if the procedure crashes between // TRUNCATE_TABLE_REMOVE_FROM_META and TRUNCATE_TABLE_CREATE_FS_LAYOUT tableDescriptor = env.getMasterServices().getTableDescriptors().get(tableName); + + // Check if we should create a recovery snapshot + if (RecoverySnapshotUtils.isRecoveryEnabled(env)) { + setNextState(TruncateTableState.TRUNCATE_TABLE_SNAPSHOT); + } else { + setNextState(TruncateTableState.TRUNCATE_TABLE_CLEAR_FS_LAYOUT); + } + break; + case TRUNCATE_TABLE_SNAPSHOT: + // Create recovery snapshot procedure as child procedure + recoverySnapshotName = RecoverySnapshotUtils.generateSnapshotName(tableName); + SnapshotProcedure snapshotProcedure = RecoverySnapshotUtils.createSnapshotProcedure(env, + tableName, recoverySnapshotName, tableDescriptor); + // Submit snapshot procedure as child procedure + addChildProcedure(snapshotProcedure); + LOG.debug("Creating recovery snapshot {} for table {} before truncation", + recoverySnapshotName, tableName); setNextState(TruncateTableState.TRUNCATE_TABLE_CLEAR_FS_LAYOUT); break; case TRUNCATE_TABLE_CLEAR_FS_LAYOUT: @@ -160,15 +178,26 @@ protected Flow executeFromState(final MasterProcedureEnv env, TruncateTableState @Override protected void rollbackState(final MasterProcedureEnv env, final TruncateTableState state) { - if (state == TruncateTableState.TRUNCATE_TABLE_PRE_OPERATION) { - // nothing to rollback, pre-truncate is just table-state checks. - // We can fail if the table does not exist or is not disabled. - // TODO: coprocessor rollback semantic is still undefined. - return; + switch (state) { + case TRUNCATE_TABLE_PRE_OPERATION: + // nothing to rollback, pre-truncate is just table-state checks. + // We can fail if the table does not exist or is not disabled. + // TODO: coprocessor rollback semantic is still undefined. + break; + case TRUNCATE_TABLE_SNAPSHOT: + // Handle recovery snapshot rollback. There is no DeleteSnapshotProcedure as such to use + // here directly as a child procedure, so we call a utility method to delete the snapshot + // which uses the SnapshotManager to delete the snapshot. + if (recoverySnapshotName != null) { + RecoverySnapshotUtils.deleteRecoverySnapshot(env, recoverySnapshotName, tableName); + recoverySnapshotName = null; + } + break; + default: + // Truncate from other states doesn't have a rollback. The execution will succeed, at some + // point. + throw new UnsupportedOperationException("unhandled state=" + state); } - - // The truncate doesn't have a rollback. The execution will succeed, at some point. - throw new UnsupportedOperationException("unhandled state=" + state); } @Override @@ -180,6 +209,7 @@ protected void completionCleanup(final MasterProcedureEnv env) { protected boolean isRollbackSupported(final TruncateTableState state) { switch (state) { case TRUNCATE_TABLE_PRE_OPERATION: + case TRUNCATE_TABLE_SNAPSHOT: return true; default: return false; @@ -244,6 +274,9 @@ protected void serializeStateData(ProcedureStateSerializer serializer) throws IO state.addRegionInfo(ProtobufUtil.toRegionInfo(hri)); } } + if (recoverySnapshotName != null) { + state.setSnapshotName(recoverySnapshotName); + } serializer.serialize(state.build()); } @@ -269,6 +302,9 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws regions.add(ProtobufUtil.toRegionInfo(hri)); } } + if (state.hasSnapshotName()) { + recoverySnapshotName = state.getSnapshotName(); + } } private static List recreateRegionInfo(final List regions) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedureWithRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedureWithRecovery.java new file mode 100644 index 000000000000..e17fb5171050 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedureWithRecovery.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestDeleteTableProcedureWithRecovery extends TestTableDDLProcedureBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDeleteTableProcedureWithRecovery.class); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupCluster() throws Exception { + // Enable recovery snapshots + UTIL.getConfiguration().setBoolean(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY, + true); + TestTableDDLProcedureBase.setupCluster(); + } + + @Test + public void testRecoverySnapshotRollback() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String[] families = new String[] { "f1", "f2" }; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with data + MasterProcedureTestingUtility.createTable(procExec, tableName, null, families); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + families); + UTIL.getAdmin().disableTable(tableName); + + // Submit the failing procedure + long procId = procExec + .submitProcedure(new FailingDeleteTableProcedure(procExec.getEnvironment(), tableName)); + + // Wait for procedure to complete (should fail) + ProcedureTestingUtility.waitProcedure(procExec, procId); + Procedure result = procExec.getResult(procId); + assertTrue("Procedure should have failed", result.isFailed()); + + // Verify no recovery snapshots remain after rollback + boolean snapshotFound = false; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + snapshotFound = true; + break; + } + } + assertTrue("Recovery snapshot should have been cleaned up during rollback", !snapshotFound); + } + + @Test + public void testRecoverySnapshotAndRestore() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName restoredTableName = TableName.valueOf(name.getMethodName() + "_restored"); + final String[] families = new String[] { "f1", "f2" }; + + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with data + MasterProcedureTestingUtility.createTable(procExec, tableName, null, families); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + families); + UTIL.getAdmin().disableTable(tableName); + + // Delete the table (this should create a recovery snapshot) + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new DeleteTableProcedure(procExec.getEnvironment(), tableName)); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + // Verify table is deleted + MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); + + // Find the recovery snapshot + String recoverySnapshotName = null; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + recoverySnapshotName = snapshot.getName(); + break; + } + } + assertTrue("Recovery snapshot should exist", recoverySnapshotName != null); + + // Restore from snapshot by cloning to a new table + UTIL.getAdmin().cloneSnapshot(recoverySnapshotName, restoredTableName); + UTIL.waitUntilAllRegionsAssigned(restoredTableName); + + // Verify restored table has original data + assertEquals(100, UTIL.countRows(restoredTableName)); + + // Clean up the cloned table + UTIL.getAdmin().disableTable(restoredTableName); + UTIL.getAdmin().deleteTable(restoredTableName); + } + + // Create a procedure that will fail after snapshot creation + public static class FailingDeleteTableProcedure extends DeleteTableProcedure { + private boolean failOnce = false; + + public FailingDeleteTableProcedure() { + super(); + } + + public FailingDeleteTableProcedure(MasterProcedureEnv env, TableName tableName) { + super(env, tableName); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, DeleteTableState state) + throws InterruptedException, ProcedureSuspendedException { + if (!failOnce && state == DeleteTableState.DELETE_TABLE_CLEAR_FS_LAYOUT) { + failOnce = true; + throw new RuntimeException("Simulated failure"); + } + return super.executeFromState(env, state); + } + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedureWithRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedureWithRecovery.java new file mode 100644 index 000000000000..1eccfc82ecda --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedureWithRecovery.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyTableState; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestModifyTableProcedureWithRecovery extends TestTableDDLProcedureBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestModifyTableProcedureWithRecovery.class); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupCluster() throws Exception { + // Enable recovery snapshots + UTIL.getConfiguration().setBoolean(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY, + true); + TestTableDDLProcedureBase.setupCluster(); + } + + @Test + public void testRecoverySnapshotRollback() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String cf1 = "cf1"; + final String cf2 = "cf2"; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with multiple column families + MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + new String[] { cf1, cf2 }); + UTIL.getAdmin().disableTable(tableName); + + // Create a procedure that will fail - modify to delete a column family + // but simulate failure after snapshot creation + // Modify table to remove cf2 (which should trigger recovery snapshot) + TableDescriptor originalHtd = UTIL.getAdmin().getDescriptor(tableName); + TableDescriptor modifiedHtd = + TableDescriptorBuilder.newBuilder(originalHtd).removeColumnFamily(cf2.getBytes()).build(); + + // Submit the failing procedure + long procId = procExec + .submitProcedure(new FailingModifyTableProcedure(procExec.getEnvironment(), modifiedHtd)); + + // Wait for procedure to complete (should fail) + ProcedureTestingUtility.waitProcedure(procExec, procId); + Procedure result = procExec.getResult(procId); + assertTrue("Procedure should have failed", result.isFailed()); + + // Verify no recovery snapshots remain after rollback + boolean snapshotFound = false; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + snapshotFound = true; + break; + } + } + assertTrue("Recovery snapshot should have been cleaned up during rollback", !snapshotFound); + } + + @Test + public void testRecoverySnapshotAndRestore() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName restoredTableName = TableName.valueOf(name.getMethodName() + "_restored"); + final String cf1 = "cf1"; + final String cf2 = "cf2"; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with multiple column families + MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + new String[] { cf1, cf2 }); + UTIL.getAdmin().disableTable(tableName); + + // Modify table to remove cf2 (which should trigger recovery snapshot) + TableDescriptor originalHtd = UTIL.getAdmin().getDescriptor(tableName); + TableDescriptor modifiedHtd = + TableDescriptorBuilder.newBuilder(originalHtd).removeColumnFamily(cf2.getBytes()).build(); + + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new ModifyTableProcedure(procExec.getEnvironment(), modifiedHtd)); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + // Verify table modification was successful + TableDescriptor currentHtd = UTIL.getAdmin().getDescriptor(tableName); + assertEquals("Should have one column family", 1, currentHtd.getColumnFamilyNames().size()); + assertTrue("Should only have cf1", currentHtd.hasColumnFamily(cf1.getBytes())); + + // Find the recovery snapshot + String recoverySnapshotName = null; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + recoverySnapshotName = snapshot.getName(); + break; + } + } + assertTrue("Recovery snapshot should exist", recoverySnapshotName != null); + + // Restore from snapshot by cloning to a new table + UTIL.getAdmin().cloneSnapshot(recoverySnapshotName, restoredTableName); + UTIL.waitUntilAllRegionsAssigned(restoredTableName); + + // Verify restored table has original structure with both column families + TableDescriptor restoredHtd = UTIL.getAdmin().getDescriptor(restoredTableName); + assertEquals("Should have two column families", 2, restoredHtd.getColumnFamilyNames().size()); + assertTrue("Should have cf1", restoredHtd.hasColumnFamily(cf1.getBytes())); + assertTrue("Should have cf2", restoredHtd.hasColumnFamily(cf2.getBytes())); + + // Clean up the cloned table + UTIL.getAdmin().disableTable(restoredTableName); + UTIL.getAdmin().deleteTable(restoredTableName); + } + + public static class FailingModifyTableProcedure extends ModifyTableProcedure { + private boolean failOnce = false; + + public FailingModifyTableProcedure() { + super(); + } + + public FailingModifyTableProcedure(MasterProcedureEnv env, TableDescriptor newTableDescriptor) + throws HBaseIOException { + super(env, newTableDescriptor); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, ModifyTableState state) + throws InterruptedException { + if (!failOnce && state == ModifyTableState.MODIFY_TABLE_CLOSE_EXCESS_REPLICAS) { + failOnce = true; + throw new RuntimeException("Simulated failure"); + } + return super.executeFromState(env, state); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRecoverySnapshotUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRecoverySnapshotUtils.java new file mode 100644 index 000000000000..57cfe57716b5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRecoverySnapshotUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, SmallTests.class }) +public class TestRecoverySnapshotUtils { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRecoverySnapshotUtils.class); + + @Test + public void testRecoverySnapshotTtlNoDescriptor() { + // Create a mock MasterProcedureEnv with a known site configuration TTL + long siteLevelTtl = 7200; // 2 hours + Configuration conf = new Configuration(); + conf.setLong(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL_KEY, siteLevelTtl); + + MasterProcedureEnv env = mock(MasterProcedureEnv.class); + when(env.getMasterConfiguration()).thenReturn(conf); + + // Test with null table descriptor - should return site configuration + long actualTtl = RecoverySnapshotUtils.getRecoverySnapshotTtl(env, null); + assertEquals("Should return site-level TTL when no table descriptor provided", siteLevelTtl, + actualTtl); + } + + @Test + public void testRecoverySnapshotTtlWithDescriptor() { + // Create a mock MasterProcedureEnv with a known site configuration TTL + long siteLevelTtl = 7200; // 2 hours + Configuration conf = new Configuration(); + conf.setLong(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL_KEY, siteLevelTtl); + + MasterProcedureEnv env = mock(MasterProcedureEnv.class); + when(env.getMasterConfiguration()).thenReturn(conf); + + // Create a table descriptor with a different TTL override + long tableLevelTtl = 3600; // 1 hour + TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf("test")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")) + .setValue(HConstants.TABLE_RECOVERY_SNAPSHOT_TTL_KEY, String.valueOf(tableLevelTtl)).build(); + + // Test with table descriptor override - should return table-level TTL + long actualTtl = RecoverySnapshotUtils.getRecoverySnapshotTtl(env, tableDescriptor); + assertEquals("Should return table-level TTL when table descriptor provides override", + tableLevelTtl, actualTtl); + } + + @Test + public void testRecoverySnapshotTtlUsesDefault() { + // Create a mock MasterProcedureEnv with default configuration (no explicit TTL set) + Configuration conf = new Configuration(); + // Don't set the TTL key, so it should use the default + + MasterProcedureEnv env = mock(MasterProcedureEnv.class); + when(env.getMasterConfiguration()).thenReturn(conf); + + // Test with null table descriptor - should return default TTL + long actualTtl = RecoverySnapshotUtils.getRecoverySnapshotTtl(env, null); + assertEquals("Should return default TTL when no site configuration provided", + HConstants.DEFAULT_SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_TTL, actualTtl); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedureWithRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedureWithRecovery.java new file mode 100644 index 000000000000..15023d48f247 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedureWithRecovery.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateRegionState; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestTruncateRegionProcedureWithRecovery extends TestTableDDLProcedureBase { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTruncateRegionProcedureWithRecovery.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestTruncateRegionProcedureWithRecovery.class); + + @Rule + public TestName name = new TestName(); + + private static void setupConf(Configuration conf) { + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); + conf.setBoolean(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY, true); + conf.setInt("hbase.client.sync.wait.timeout.msec", 60000); + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(3); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + + // Turn off balancer, so it doesn't cut in and mess up our placements. + UTIL.getAdmin().balancerSwitch(false, true); + // Turn off the meta scanner, so it doesn't remove, parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + } + + @After + public void tearDown() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) { + UTIL.deleteTable(htd.getTableName()); + } + } + + @Test + public void testRecoverySnapshotRollback() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String[] families = new String[] { "f1", "f2" }; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with split keys + final byte[][] splitKeys = new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60") }; + MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, families); + + // Insert data + insertData(UTIL, tableName, 2, 20, families); + insertData(UTIL, tableName, 2, 31, families); + insertData(UTIL, tableName, 2, 61, families); + + // Get a region to truncate + MasterProcedureEnv environment = procExec.getEnvironment(); + RegionInfo regionToTruncate = environment.getAssignmentManager().getAssignedRegions().stream() + .filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString())) + .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get(); + + // Create a procedure that might fail. Use a simple approach that creates a custom procedure + // that fails after snapshot. + // Submit the failing procedure + long procId = + procExec.submitProcedure(new FailingTruncateRegionProcedure(environment, regionToTruncate)); + + // Wait for procedure to complete (should fail) + ProcedureTestingUtility.waitProcedure(procExec, procId); + Procedure result = procExec.getResult(procId); + assertTrue("Procedure should have failed", result.isFailed()); + + // Verify no recovery snapshots remain after rollback + boolean snapshotFound = false; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + snapshotFound = true; + break; + } + } + assertTrue("Recovery snapshot should have been cleaned up during rollback", !snapshotFound); + } + + @Test + public void testRecoverySnapshotAndRestore() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName restoredTableName = TableName.valueOf(name.getMethodName() + "_restored"); + final String[] families = new String[] { "f1", "f2" }; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with split keys + final byte[][] splitKeys = new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60") }; + MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, families); + + // Insert data + insertData(UTIL, tableName, 2, 20, families); + insertData(UTIL, tableName, 2, 31, families); + insertData(UTIL, tableName, 2, 61, families); + int initialRowCount = UTIL.countRows(tableName); + + // Get a region to truncate + MasterProcedureEnv environment = procExec.getEnvironment(); + RegionInfo regionToTruncate = environment.getAssignmentManager().getAssignedRegions().stream() + .filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString())) + .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get(); + + // Truncate the region (this should create a recovery snapshot) + long procId = + procExec.submitProcedure(new TruncateRegionProcedure(environment, regionToTruncate)); + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + // Verify region is truncated (should have fewer rows) + int rowsAfterTruncate = UTIL.countRows(tableName); + assertTrue("Should have fewer rows after truncate", rowsAfterTruncate < initialRowCount); + + // Find the recovery snapshot + String recoverySnapshotName = null; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + recoverySnapshotName = snapshot.getName(); + break; + } + } + assertTrue("Recovery snapshot should exist", recoverySnapshotName != null); + + // Restore from snapshot by cloning to a new table + UTIL.getAdmin().cloneSnapshot(recoverySnapshotName, restoredTableName); + UTIL.waitUntilAllRegionsAssigned(restoredTableName); + + // Verify restored table has original data + assertEquals("Restored table should have original data", initialRowCount, + UTIL.countRows(restoredTableName)); + + // Clean up the cloned table + UTIL.getAdmin().disableTable(restoredTableName); + UTIL.getAdmin().deleteTable(restoredTableName); + } + + public static class FailingTruncateRegionProcedure extends TruncateRegionProcedure { + private boolean failOnce = false; + + public FailingTruncateRegionProcedure() { + super(); + } + + public FailingTruncateRegionProcedure(MasterProcedureEnv env, RegionInfo region) + throws HBaseIOException { + super(env, region); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, TruncateRegionState state) + throws InterruptedException { + if (!failOnce && state == TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE) { + failOnce = true; + throw new RuntimeException("Simulated failure"); + } + return super.executeFromState(env, state); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedureWithRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedureWithRecovery.java new file mode 100644 index 000000000000..34ffabc58548 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedureWithRecovery.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateTableState; + +@Category({ MasterTests.class, LargeTests.class }) +public class TestTruncateTableProcedureWithRecovery extends TestTableDDLProcedureBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTruncateTableProcedureWithRecovery.class); + + @Rule + public TestName name = new TestName(); + + @BeforeClass + public static void setupCluster() throws Exception { + // Enable recovery snapshots + UTIL.getConfiguration().setBoolean(HConstants.SNAPSHOT_BEFORE_DESTRUCTIVE_ACTION_ENABLED_KEY, + true); + TestTableDDLProcedureBase.setupCluster(); + } + + @Test + public void testRecoverySnapshotRollback() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String[] families = new String[] { "f1", "f2" }; + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + // Create table with data + MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, null, + families); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + families); + assertEquals(100, UTIL.countRows(tableName)); + + // Disable the table + UTIL.getAdmin().disableTable(tableName); + + // Submit the failing procedure + long procId = procExec.submitProcedure( + new FailingTruncateTableProcedure(procExec.getEnvironment(), tableName, false)); + + // Wait for procedure to complete (should fail) + ProcedureTestingUtility.waitProcedure(procExec, procId); + Procedure result = procExec.getResult(procId); + assertTrue("Procedure should have failed", result.isFailed()); + + // Verify no recovery snapshots remain after rollback + boolean snapshotFound = false; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + snapshotFound = true; + break; + } + } + assertTrue("Recovery snapshot should have been cleaned up during rollback", !snapshotFound); + } + + @Test + public void testRecoverySnapshotAndRestore() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final TableName restoredTableName = TableName.valueOf(name.getMethodName() + "_restored"); + final String[] families = new String[] { "f1", "f2" }; + + // Create table with data + MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, null, + families); + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, new byte[0][], + families); + assertEquals(100, UTIL.countRows(tableName)); + + // Disable the table + UTIL.getAdmin().disableTable(tableName); + + // Truncate the table (this should create a recovery snapshot) + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new TruncateTableProcedure(procExec.getEnvironment(), tableName, false)); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + // Verify table is truncated + UTIL.waitUntilAllRegionsAssigned(tableName); + assertEquals(0, UTIL.countRows(tableName)); + + // Find the recovery snapshot + String recoverySnapshotName = null; + for (SnapshotDescription snapshot : UTIL.getAdmin().listSnapshots()) { + if (snapshot.getName().startsWith("auto_" + tableName.getNameAsString())) { + recoverySnapshotName = snapshot.getName(); + break; + } + } + assertTrue("Recovery snapshot should exist", recoverySnapshotName != null); + + // Restore from snapshot by cloning to a new table + UTIL.getAdmin().cloneSnapshot(recoverySnapshotName, restoredTableName); + UTIL.waitUntilAllRegionsAssigned(restoredTableName); + + // Verify restored table has original data + assertEquals(100, UTIL.countRows(restoredTableName)); + + // Clean up the cloned table + UTIL.getAdmin().disableTable(restoredTableName); + UTIL.getAdmin().deleteTable(restoredTableName); + } + + public static class FailingTruncateTableProcedure extends TruncateTableProcedure { + private boolean failOnce = false; + + public FailingTruncateTableProcedure() { + super(); + } + + public FailingTruncateTableProcedure(MasterProcedureEnv env, TableName tableName, + boolean preserveSplits) throws HBaseIOException { + super(env, tableName, preserveSplits); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, TruncateTableState state) + throws InterruptedException { + if (!failOnce && state == TruncateTableState.TRUNCATE_TABLE_CLEAR_FS_LAYOUT) { + failOnce = true; + throw new RuntimeException("Simulated failure"); + } + return super.executeFromState(env, state); + } + } +} From 73288432452aead6722fd1eac81643bc684b2e9a Mon Sep 17 00:00:00 2001 From: wangxiangdong123 <810410559@qq.com> Date: Tue, 5 Aug 2025 19:52:05 +0800 Subject: [PATCH 016/336] HBASE-29496 Fix Javadoc typo: 'DsiableTableProcedure' should be 'DisableTableProcedure' (#7190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nihal Jain Signed-off-by: Dávid Paksy (cherry picked from commit 55cf6e22d6cb328f43f1ee9a7a1d41fa0d26ef1f) --- .../hadoop/hbase/master/assignment/AssignmentManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 3521353e8843..0bc651b4954c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1095,7 +1095,7 @@ private int submitUnassignProcedure(TableName tableName, } /** - * Called by DsiableTableProcedure to unassign all regions for a table. Will skip submit unassign + * Called by DisableTableProcedure to unassign all regions for a table. Will skip submit unassign * procedure if the region is in transition, so you may need to call this method multiple times. * @param tableName the table for closing excess region replicas * @param submit for submitting procedure From 058fe41c72669cc6b8c93dff8830099c18196a27 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Mon, 11 Aug 2025 14:25:46 +0200 Subject: [PATCH 017/336] HBASE-29508 Define HBase specific TLS config properties for InfoServer (#7204) Signed-off-by: Nihal Jain (cherry picked from commit 70b49d7ae6c49b011d57c60db3f4918bcbad5a32) --- .../apache/hadoop/hbase/http/InfoServer.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index 6a08e21df97d..ea73be808f07 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -42,6 +42,9 @@ public class InfoServer { private static final String HBASE_APP_DIR = "hbase-webapps"; private final org.apache.hadoop.hbase.http.HttpServer httpServer; + private static final String HADOOP_WEB_TLS_CONFIG_PREFIX = "ssl.server."; + private static final String HBASE_WEB_TLS_CONFIG_PREFIX = "hbase.ui.ssl."; + /** * Create a status server on the given port. The jsp scripts are taken from * src/hbase-webapps/name. @@ -70,19 +73,16 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, // We are using the Hadoop HTTP server config properties. // This makes it easy to keep in sync with Hadoop's UI servers, but hard to set this // separately for HBase. - builder - .keyPassword(HBaseConfiguration.getPassword(c, "ssl.server.keystore.keypassword", null)) - .keyStore(c.get("ssl.server.keystore.location"), - HBaseConfiguration.getPassword(c, "ssl.server.keystore.password", null), - c.get("ssl.server.keystore.type", "jks")) - .trustStore(c.get("ssl.server.truststore.location"), - HBaseConfiguration.getPassword(c, "ssl.server.truststore.password", null), - c.get("ssl.server.truststore.type", "jks")) + builder.keyPassword(getTLSPassword(c, "keystore.keypassword")) + .keyStore(getTLSProperty(c, "keystore.location"), getTLSPassword(c, "keystore.password"), + getTLSProperty(c, "keystore.type", "jks")) + .trustStore(getTLSProperty(c, "truststore.location"), + getTLSPassword(c, "truststore.password"), getTLSProperty(c, "truststore.type", "jks")) // The ssl.server.*.protocols properties do not exist in Hadoop at the time of writing. - .setIncludeProtocols(c.get("ssl.server.include.protocols")) - .setExcludeProtocols(c.get("ssl.server.exclude.protocols")) - .setIncludeCiphers(c.get("ssl.server.include.cipher.list")) - .setExcludeCiphers(c.get("ssl.server.exclude.cipher.list")); + .setIncludeProtocols(getTLSProperty(c, "include.protocols")) + .setExcludeProtocols(getTLSProperty(c, "exclude.protocols")) + .setIncludeCiphers(getTLSProperty(c, "include.cipher.list")) + .setExcludeCiphers(getTLSProperty(c, "exclude.cipher.list")); } final String httpAuthType = c.get(HttpServer.HTTP_UI_AUTHENTICATION, "").toLowerCase(); @@ -104,6 +104,20 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, this.httpServer = builder.build(); } + private String getTLSPassword(Configuration c, String postfix) throws IOException { + return HBaseConfiguration.getPassword(c, HBASE_WEB_TLS_CONFIG_PREFIX + postfix, + HBaseConfiguration.getPassword(c, HADOOP_WEB_TLS_CONFIG_PREFIX + postfix, null)); + } + + private String getTLSProperty(Configuration c, String postfix) { + return getTLSProperty(c, postfix, null); + } + + private String getTLSProperty(Configuration c, String postfix, String defaultValue) { + return c.get(HBASE_WEB_TLS_CONFIG_PREFIX + postfix, + c.get(HADOOP_WEB_TLS_CONFIG_PREFIX + postfix, defaultValue)); + } + /** * Builds an ACL that will restrict the users who can issue commands to endpoints on the UI which * are meant only for administrators. From 5bc9265f598a5b6045ef59dcaa0f090311d1b251 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Tue, 12 Aug 2025 06:30:33 -0700 Subject: [PATCH 018/336] HBASE-29469 Add metrics with more detail for RpcThrottlingExceptions (#7214) Co-authored-by: skhillon Signed-off by: cconnell Reviewed by: kgeisz --- .../quotas/RegionServerRpcQuotaManager.java | 8 + .../regionserver/MetricsRegionServer.java | 16 + .../metrics/MetricsThrottleExceptions.java | 80 +++++ .../regionserver/TestMetricsRegionServer.java | 42 +++ .../TestMetricsThrottleExceptions.java | 294 ++++++++++++++++++ 5 files changed, 440 insertions(+) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 03fbfde47a13..958793dcdf00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -196,6 +196,10 @@ public OperationQuota checkScanQuota(final Region region, } catch (RpcThrottlingException e) { LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " scan=" + scanRequest.getScannerId() + ": " + e.getMessage()); + + rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + table.getNameAsString()); + throw e; } return quota; @@ -269,6 +273,10 @@ public OperationQuota checkBatchQuota(final Region region, final int numWrites, } catch (RpcThrottlingException e) { LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " numWrites=" + numWrites + " numReads=" + numReads + ": " + e.getMessage()); + + rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + table.getNameAsString()); + throw e; } return quota; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index a0bf25dc2eaa..580f77874992 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.metrics.MetricRegistries; import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.hadoop.hbase.regionserver.metrics.MetricsThrottleExceptions; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -47,6 +49,7 @@ public class MetricsRegionServer { private MetricsRegionServerQuotaSource quotaSource; private MetricRegistry metricRegistry; + private MetricsThrottleExceptions throttleMetrics; private Timer bulkLoadTimer; // Incremented once for each call to Scan#nextRaw private Meter serverReadQueryMeter; @@ -78,6 +81,8 @@ public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, Confi serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond"); serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond"); } + + throttleMetrics = new MetricsThrottleExceptions(metricRegistry); } MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, @@ -296,4 +301,15 @@ public void incrScannerLeaseExpired() { serverSource.incrScannerLeaseExpired(); } + /** + * Record a throttle exception with contextual information. + * @param throttleType the type of throttle exception from RpcThrottlingException.Type enum + * @param user the user who triggered the throttle + * @param table the table that was being accessed + */ + public void recordThrottleException(RpcThrottlingException.Type throttleType, String user, + String table) { + throttleMetrics.recordThrottleException(throttleType, user, table); + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java new file mode 100644 index 000000000000..ddf451ab2e29 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class MetricsThrottleExceptions { + + /** + * The name of the metrics + */ + private static final String METRICS_NAME = "ThrottleExceptions"; + + /** + * The name of the metrics context that metrics will be under. + */ + private static final String METRICS_CONTEXT = "regionserver"; + + /** + * Description + */ + private static final String METRICS_DESCRIPTION = "Metrics about RPC throttling exceptions"; + + /** + * The name of the metrics context that metrics will be under in jmx + */ + private static final String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME; + + private final MetricRegistry registry; + + public MetricsThrottleExceptions(MetricRegistry sharedRegistry) { + registry = sharedRegistry; + } + + /** + * Record a throttle exception with contextual information. + * @param throttleType the type of throttle exception + * @param user the user who triggered the throttle + * @param table the table that was being accessed + */ + public void recordThrottleException(RpcThrottlingException.Type throttleType, String user, + String table) { + String metricName = qualifyThrottleMetric(throttleType, user, table); + registry.counter(metricName).increment(); + } + + private static String qualifyThrottleMetric(RpcThrottlingException.Type throttleType, String user, + String table) { + return String.format("RpcThrottlingException_Type_%s_User_%s_Table_%s", throttleType.name(), + sanitizeMetricName(user), sanitizeMetricName(table)); + } + + private static String sanitizeMetricName(String name) { + if (name == null) { + return "unknown"; + } + // Only replace characters that are problematic for JMX ObjectNames + // Keep meaningful characters like hyphens, periods, etc. + return name.replaceAll("[,=:*?\"\\n]", "_"); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index 2186aa7cc4dc..ce78aa40ab7a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -26,11 +26,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.JvmPauseMonitor; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -68,6 +71,12 @@ public void setUp() { serverSource = rsm.getMetricsSource(); } + @After + public void tearDown() { + // Clean up global registries after each test to avoid interference + MetricRegistries.global().clear(); + } + @Test public void testWrapperSource() { HELPER.assertTag("serverName", "test", serverSource); @@ -314,4 +323,37 @@ public void testScannerMetrics() { HELPER.assertGauge("activeScanners", 0, serverSource); } + @Test + public void testThrottleExceptionMetricsIntegration() { + // Record different types of throttle exceptions + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + rsm.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", "logs"); + rsm.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, "charlie", + "metadata"); + + // Record the same exception multiple times to test increment + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + + // Verify the specific counters were created and have correct values using HELPER + HELPER.assertCounter("RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", + 3L, serverSource); + HELPER.assertCounter("RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_logs", 1L, + serverSource); + HELPER.assertCounter("RpcThrottlingException_Type_ReadSizeExceeded_User_charlie_Table_metadata", + 1L, serverSource); + + // Test metric name sanitization through the integration + rsm.recordThrottleException(RpcThrottlingException.Type.RequestSizeExceeded, + "user.with@special", "table:with,problematic=chars"); + HELPER.assertCounter( + "RpcThrottlingException_Type_RequestSizeExceeded_User_user.with@special_Table_table_with_problematic_chars", + 1L, serverSource); + + // Test null handling through the integration + rsm.recordThrottleException(RpcThrottlingException.Type.ReadCapacityUnitExceeded, null, null); + HELPER.assertCounter( + "RpcThrottlingException_Type_ReadCapacityUnitExceeded_User_unknown_Table_unknown", 1L, + serverSource); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java new file mode 100644 index 000000000000..0fa02c42a325 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestMetricsThrottleExceptions { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetricsThrottleExceptions.class); + + private MetricRegistry testRegistry; + private MetricsThrottleExceptions throttleMetrics; + + @After + public void cleanup() { + // Clean up global registries after each test to avoid interference + MetricRegistries.global().clear(); + } + + @Test + public void testBasicThrottleMetricsRecording() { + setupTestMetrics(); + + // Record a throttle exception + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + + // Verify the counter exists and has correct value + Optional metric = + testRegistry.get("RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users"); + assertTrue("Counter metric should be present", metric.isPresent()); + assertTrue("Metric should be a counter", metric.get() instanceof Counter); + + Counter counter = (Counter) metric.get(); + assertEquals("Counter should have count of 1", 1, counter.getCount()); + } + + @Test + public void testMultipleThrottleTypes() { + setupTestMetrics(); + + // Record different types of throttle exceptions + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", + "logs"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, "charlie", + "metadata"); + + // Verify all three counters were created + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", 1); + verifyCounter(testRegistry, "RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_logs", + 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_ReadSizeExceeded_User_charlie_Table_metadata", 1); + } + + @Test + public void testCounterIncrement() { + setupTestMetrics(); + + // Record the same throttle exception multiple times + String metricName = "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users"; + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + + // Verify the counter incremented correctly + verifyCounter(testRegistry, metricName, 3); + } + + @Test + public void testMetricNameSanitization() { + setupTestMetrics(); + + // Test that meaningful characters are preserved (hyphens, periods, etc.) + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, + "user.name@company", "my-table-prod"); + + // Verify meaningful characters are preserved, only JMX-problematic chars are replaced + String expectedMetricName = + "RpcThrottlingException_Type_WriteSizeExceeded_User_user.name@company_Table_my-table-prod"; + verifyCounter(testRegistry, expectedMetricName, 1); + + // Test that JMX-problematic characters are sanitized + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, + "user,with=bad:chars*", "table?with\"quotes"); + String problematicMetricName = + "RpcThrottlingException_Type_ReadSizeExceeded_User_user_with_bad_chars__Table_table_with_quotes"; + verifyCounter(testRegistry, problematicMetricName, 1); + } + + @Test + public void testNullHandling() { + setupTestMetrics(); + + // Test null user and table names + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, null, + null); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "alice", + null); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, null, + "users"); + + // Verify null values are replaced with "unknown" + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_unknown_Table_unknown", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_WriteSizeExceeded_User_alice_Table_unknown", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_ReadSizeExceeded_User_unknown_Table_users", 1); + } + + @Test + public void testConcurrentAccess() throws InterruptedException { + setupTestMetrics(); + + int numThreads = 10; + int incrementsPerThread = 100; + + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(numThreads); + AtomicInteger exceptions = new AtomicInteger(0); + + // Create multiple threads that increment the same counter concurrently + for (int i = 0; i < numThreads; i++) { + executor.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < incrementsPerThread; j++) { + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + } + } catch (Exception e) { + exceptions.incrementAndGet(); + } finally { + doneLatch.countDown(); + } + }); + } + + // Start all threads at once + startLatch.countDown(); + + // Wait for all threads to complete + boolean completed = doneLatch.await(30, TimeUnit.SECONDS); + assertTrue("All threads should complete within timeout", completed); + assertEquals("No exceptions should occur during concurrent access", 0, exceptions.get()); + + // Verify the final counter value + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", + numThreads * incrementsPerThread); + + executor.shutdown(); + } + + @Test + public void testCommonTableNamePatterns() { + setupTestMetrics(); + + // Test common HBase table name patterns that should be preserved + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "service-user", "my-app-logs"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, + "batch.process", "namespace:table-name"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, + "user_123", "test_table_v2"); + + // Verify common patterns are preserved correctly (note: colon gets replaced with underscore) + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_service-user_Table_my-app-logs", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_WriteSizeExceeded_User_batch.process_Table_namespace_table-name", + 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_ReadSizeExceeded_User_user_123_Table_test_table_v2", 1); + } + + @Test + public void testAllThrottleExceptionTypes() { + setupTestMetrics(); + + // Test all 13 throttle exception types from RpcThrottlingException.Type enum + RpcThrottlingException.Type[] throttleTypes = RpcThrottlingException.Type.values(); + + // Record one exception for each type + for (RpcThrottlingException.Type throttleType : throttleTypes) { + throttleMetrics.recordThrottleException(throttleType, "testuser", "testtable"); + } + + // Verify all counters were created with correct values + for (RpcThrottlingException.Type throttleType : throttleTypes) { + String expectedMetricName = + "RpcThrottlingException_Type_" + throttleType.name() + "_User_testuser_Table_testtable"; + verifyCounter(testRegistry, expectedMetricName, 1); + } + } + + @Test + public void testMultipleInstances() { + setupTestMetrics(); + + // Test that multiple instances of MetricsThrottleExceptions work with the same registry + MetricsThrottleExceptions metrics1 = new MetricsThrottleExceptions(testRegistry); + MetricsThrottleExceptions metrics2 = new MetricsThrottleExceptions(testRegistry); + + // Record different exceptions on each instance + metrics1.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", + "table1"); + metrics2.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", + "table2"); + + // Verify both counters exist in the shared registry + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_table1", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_table2", 1); + } + + /** + * Helper method to set up test metrics registry and instance + */ + private void setupTestMetrics() { + MetricRegistryInfo registryInfo = getRegistryInfo(); + testRegistry = MetricRegistries.global().create(registryInfo); + throttleMetrics = new MetricsThrottleExceptions(testRegistry); + } + + /** + * Helper method to verify a counter exists and has the expected value + */ + private void verifyCounter(MetricRegistry registry, String metricName, long expectedCount) { + Optional metric = registry.get(metricName); + assertTrue("Counter metric '" + metricName + "' should be present", metric.isPresent()); + assertTrue("Metric should be a counter", metric.get() instanceof Counter); + + Counter counter = (Counter) metric.get(); + assertEquals("Counter '" + metricName + "' should have expected count", expectedCount, + counter.getCount()); + } + + /** + * Helper method to create the expected MetricRegistryInfo for ThrottleExceptions + */ + private MetricRegistryInfo getRegistryInfo() { + return new MetricRegistryInfo("ThrottleExceptions", "Metrics about RPC throttling exceptions", + "RegionServer,sub=ThrottleExceptions", "regionserver", false); + } +} From 11016018457687f598e99a259ad37bce7cd56723 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 14 Aug 2025 15:28:30 +0800 Subject: [PATCH 019/336] =?UTF-8?q?HBASE-29463=20Bidirectional=20serial=20?= =?UTF-8?q?replication=20will=20block=20if=20a=20region=E2=80=99s=20last?= =?UTF-8?q?=20edit=20before=20rs=20crashed=20was=20from=20the=20peer=20clu?= =?UTF-8?q?ster=20(#7172)=20(#7225)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit bea4272960e0c3a92eaf436ec6cc1c5a2527ffc3) Signed-off-by: Nick Dimiduk --- .../replication/ChainWALEntryFilter.java | 7 ++ .../ClusterMarkingEntryFilter.java | 4 +- .../replication/ScopeWALEntryFilter.java | 16 ++-- .../hbase/replication/WALEntryFilter.java | 14 ++++ .../hbase/replication/WALEntryFilterBase.java | 66 ++++++++++++++++ .../regionserver/ReplicationSource.java | 1 + .../SerialReplicationSourceWALReader.java | 16 ++-- ...TestBidirectionSerialReplicationStuck.java | 79 +++++++++++++++++++ .../replication/TestReplicationBase.java | 31 ++++++-- .../TestReplicationWALEntryFilters.java | 11 ++- 10 files changed, 218 insertions(+), 27 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilterBase.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestBidirectionSerialReplicationStuck.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java index aa84f4705b0f..9683472f3bab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java @@ -53,6 +53,13 @@ public ChainWALEntryFilter(List filters) { initCellFilters(); } + @Override + public void setSerial(boolean serial) { + for (WALEntryFilter filter : filters) { + filter.setSerial(serial); + } + } + public void initCellFilters() { ArrayList cellFilters = new ArrayList<>(filters.length); for (WALEntryFilter filter : filters) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java index e05e79eab5a3..041b9798857b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java @@ -31,7 +31,7 @@ */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION) @InterfaceStability.Evolving -public class ClusterMarkingEntryFilter implements WALEntryFilter { +public class ClusterMarkingEntryFilter extends WALEntryFilterBase { private UUID clusterId; private UUID peerClusterId; private ReplicationEndpoint replicationEndpoint; @@ -64,6 +64,6 @@ public Entry filter(Entry entry) { return entry; } } - return null; + return clearOrNull(entry); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java index 6dc41bcc014a..1429379deb3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ScopeWALEntryFilter.java @@ -26,24 +26,26 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.base.Predicate; +import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils; /** * Keeps KVs that are scoped other than local */ @InterfaceAudience.Private -public class ScopeWALEntryFilter implements WALEntryFilter, WALCellFilter { +public class ScopeWALEntryFilter extends WALEntryFilterBase implements WALCellFilter { private final BulkLoadCellFilter bulkLoadFilter = new BulkLoadCellFilter(); @Override public Entry filter(Entry entry) { - // Do not filter out an entire entry by replication scopes. As now we support serial - // replication, the sequence id of a marker is also needed by upper layer. We will filter out - // all the cells in the filterCell method below if the replication scopes is null or empty. - return entry; + NavigableMap scopes = entry.getKey().getReplicationScopes(); + if (MapUtils.isNotEmpty(scopes)) { + return entry; + } + return clearOrNull(entry); } - private boolean hasGlobalScope(NavigableMap scopes, byte[] family) { + private static boolean hasGlobalScope(NavigableMap scopes, byte[] family) { Integer scope = scopes.get(family); return scope != null && scope.intValue() == HConstants.REPLICATION_SCOPE_GLOBAL; } @@ -51,7 +53,7 @@ private boolean hasGlobalScope(NavigableMap scopes, byte[] fami @Override public Cell filterCell(Entry entry, Cell cell) { NavigableMap scopes = entry.getKey().getReplicationScopes(); - if (scopes == null || scopes.isEmpty()) { + if (MapUtils.isEmpty(scopes)) { return null; } byte[] family = CellUtil.cloneFamily(cell); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilter.java index 8aa60f74ebba..d77fedd94bca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilter.java @@ -50,4 +50,18 @@ public interface WALEntryFilter { * the entry to be skipped for replication. */ Entry filter(Entry entry); + + /** + * Tell the filter whether the peer is a serial replication peer. + *

+ * For serial replication, usually you should not filter out an entire entry, unless the peer + * config does not contain the table, because we need the region name and sequence id of the entry + * to advance the pushed sequence id, otherwise the replication may be blocked. You can just + * filter out all the cells of the entry to stop it being replicated to peer cluster,or just rely + * on the {@link WALCellFilter#filterCell(Entry, org.apache.hadoop.hbase.Cell)} method to filter + * all the cells out. + * @param serial {@code true} if the peer is a serial replication peer, otherwise {@code false} + */ + default void setSerial(boolean serial) { + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilterBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilterBase.java new file mode 100644 index 000000000000..81efae0e7a82 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/WALEntryFilterBase.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Base class for {@link WALEntryFilter}, store the necessary common properties like + * {@link #serial}. + *

+ * Why need to treat serial replication specially: + *

+ * Under some special cases, we may filter out some entries but we still need to record the last + * pushed sequence id for these entries. For example, when we setup a bidirection replication A + * <-> B, if we write to both cluster A and cluster B, cluster A will not replicate the + * entries which are replicated from cluster B, which means we may have holes in the replication + * sequence ids. So if the region is closed abnormally, i.e, we do not have a close event for the + * region, and before the closing, we have some entries from cluster B, then the replication from + * cluster A to cluster B will be stuck if we do not record the last pushed sequence id of these + * entries because we will find out that the previous sequence id range will never finish. So we + * need to record the sequence id for these entries so the last pushed sequence id can reach the + * region barrier. + * @see HBASE-29463 + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION) +public abstract class WALEntryFilterBase implements WALEntryFilter { + + protected boolean serial; + + @Override + public void setSerial(boolean serial) { + this.serial = serial; + } + + /** + * Call this method when you do not need to replicate the entry. + *

+ * For serial replication, since still need to WALKey for recording progress, we clear all the + * cells of the WALEdit. For normal replication, we just return null. + */ + protected final Entry clearOrNull(Entry entry) { + if (serial) { + entry.getEdit().getCells().clear(); + return entry; + } else { + return null; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index aebdccdc92dc..2ce155125540 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -332,6 +332,7 @@ private void initializeWALEntryFilter(UUID peerClusterId) { } filters.add(new ClusterMarkingEntryFilter(clusterId, peerClusterId, replicationEndpoint)); this.walEntryFilter = new ChainWALEntryFilter(filters); + this.walEntryFilter.setSerial(replicationPeer.getPeerConfig().isSerial()); } private void tryStartNewShipper(String walGroupId) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationSourceWALReader.java index 41d95df28219..d1a2e8b57340 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationSourceWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationSourceWALReader.java @@ -97,17 +97,14 @@ protected void readWALEntries(WALEntryStream entryStream, WALEntryBatch batch) } sleepMultiplier = sleep(sleepMultiplier); } - // arrive here means we can push the entry, record the last sequence id - batch.setLastSeqId(Bytes.toString(entry.getKey().getEncodedRegionName()), - entry.getKey().getSequenceId()); // actually remove the entry. - removeEntryFromStream(entryStream, batch); + removeEntryFromStream(entry, entryStream, batch); if (addEntryToBatch(batch, entry)) { break; } } else { // actually remove the entry. - removeEntryFromStream(entryStream, batch); + removeEntryFromStream(null, entryStream, batch); } WALEntryStream.HasNext hasNext = entryStream.hasNext(); // always return if we have switched to a new file. @@ -125,9 +122,14 @@ protected void readWALEntries(WALEntryStream entryStream, WALEntryBatch batch) } } - private void removeEntryFromStream(WALEntryStream entryStream, WALEntryBatch batch) { + private void removeEntryFromStream(Entry entry, WALEntryStream entryStream, WALEntryBatch batch) { entryStream.next(); - firstCellInEntryBeforeFiltering = null; batch.setLastWalPosition(entryStream.getPosition()); + // record last pushed sequence id if needed + if (entry != null) { + batch.setLastSeqId(Bytes.toString(entry.getKey().getEncodedRegionName()), + entry.getKey().getSequenceId()); + } + firstCellInEntryBeforeFiltering = null; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestBidirectionSerialReplicationStuck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestBidirectionSerialReplicationStuck.java new file mode 100644 index 000000000000..f069d6b1095b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestBidirectionSerialReplicationStuck.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ReplicationTests.class, LargeTests.class }) +public class TestBidirectionSerialReplicationStuck extends TestReplicationBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBidirectionSerialReplicationStuck.class); + + @Override + protected boolean isSerialPeer() { + return true; + } + + @Override + public void setUpBase() throws Exception { + UTIL1.ensureSomeRegionServersAvailable(2); + hbaseAdmin.balancerSwitch(false, true); + addPeer(PEER_ID2, tableName, UTIL1, UTIL2); + addPeer(PEER_ID2, tableName, UTIL2, UTIL1); + } + + @Override + public void tearDownBase() throws Exception { + removePeer(PEER_ID2, UTIL1); + removePeer(PEER_ID2, UTIL2); + } + + @Test + public void testStuck() throws Exception { + // disable the peer cluster1 -> cluster2 + hbaseAdmin.disableReplicationPeer(PEER_ID2); + byte[] qualifier = Bytes.toBytes("q"); + htable1.put(new Put(Bytes.toBytes("aaa-1")).addColumn(famName, qualifier, Bytes.toBytes(1))); + + // add a row to cluster2 and wait it replicate back to cluster1 + htable2.put(new Put(Bytes.toBytes("aaa-2")).addColumn(famName, qualifier, Bytes.toBytes(2))); + UTIL1.waitFor(30000, () -> htable1.exists(new Get(Bytes.toBytes("aaa-2")))); + + // kill the region server which holds the region which contains our rows + UTIL1.getRSForFirstRegionInTable(tableName).abort("for testing"); + // wait until the region is online + UTIL1.waitFor(30000, () -> htable1.exists(new Get(Bytes.toBytes("aaa-2")))); + + // put a new row in cluster1 + htable1.put(new Put(Bytes.toBytes("aaa-3")).addColumn(famName, qualifier, Bytes.toBytes(3))); + + // enable peer cluster1 -> cluster2, the new row should be replicated to cluster2 + hbaseAdmin.enableReplicationPeer(PEER_ID2); + UTIL1.waitFor(30000, () -> htable2.exists(new Get(Bytes.toBytes("aaa-3")))); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java index fba25aee4a39..b021dcbc0d40 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -272,16 +272,27 @@ public static void setUpBeforeClass() throws Exception { } private boolean peerExist(String peerId) throws IOException { - return hbaseAdmin.listReplicationPeers().stream().anyMatch(p -> peerId.equals(p.getPeerId())); + return peerExist(peerId, UTIL1); + } + + private boolean peerExist(String peerId, HBaseTestingUtility util) throws IOException { + return util.getAdmin().listReplicationPeers().stream() + .anyMatch(p -> peerId.equals(p.getPeerId())); } protected final void addPeer(String peerId, TableName tableName) throws Exception { - if (!peerExist(peerId)) { - ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder() - .setClusterKey(UTIL2.getClusterKey()).setSerial(isSerialPeer()) - .setReplicationEndpointImpl(ReplicationEndpointTest.class.getName()); - hbaseAdmin.addReplicationPeer(peerId, builder.build()); + addPeer(peerId, tableName, UTIL1, UTIL2); + } + + protected final void addPeer(String peerId, TableName tableName, HBaseTestingUtility source, + HBaseTestingUtility target) throws Exception { + if (peerExist(peerId, source)) { + return; } + ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder() + .setClusterKey(target.getClusterKey()).setSerial(isSerialPeer()) + .setReplicationEndpointImpl(ReplicationEndpointTest.class.getName()); + source.getAdmin().addReplicationPeer(peerId, builder.build()); } @Before @@ -290,8 +301,12 @@ public void setUpBase() throws Exception { } protected final void removePeer(String peerId) throws Exception { - if (peerExist(peerId)) { - hbaseAdmin.removeReplicationPeer(peerId); + removePeer(peerId, UTIL1); + } + + protected final void removePeer(String peerId, HBaseTestingUtility util) throws Exception { + if (peerExist(peerId, util)) { + util.getAdmin().removeReplicationPeer(peerId); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index 1e26a940b2f8..762945d745ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -113,15 +113,20 @@ public void testScopeWALEntryFilter() { Entry userEntryEmpty = createEntry(null); // no scopes - // now we will not filter out entries without a replication scope since serial replication still - // need the sequence id, but the cells will all be filtered out. + assertNull(filter.filter(userEntry)); + // now for serial replication, we will not filter out entries without a replication scope since + // serial replication still need the sequence id, but the cells will all be filtered out. + filter.setSerial(true); assertTrue(filter.filter(userEntry).getEdit().isEmpty()); + filter.setSerial(false); // empty scopes - // ditto TreeMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); userEntry = createEntry(scopes, a, b); + assertNull(filter.filter(userEntry)); + filter.setSerial(true); assertTrue(filter.filter(userEntry).getEdit().isEmpty()); + filter.setSerial(false); // different scope scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); From 704bcce69e06d98d7242d7f74ce06aedd718e420 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 11:25:10 +0800 Subject: [PATCH 020/336] HBASE-29527 Bump org.bouncycastle:bcpkix-jdk18on from 1.78 to 1.81 (#7223) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.78 to 1.81. - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) --- updated-dependencies: - dependency-name: org.bouncycastle:bcpkix-jdk18on dependency-version: 1.81 dependency-type: direct:production --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Duo Zhang Signed-off-by: Pankaj Kumar Signed-off-by: Nihal Jain Signed-off-by: Dávid Paksy Signed-off-by: Duo Zhang (cherry picked from commit e40ba229b411fe2acb23374ce538586727fb1d59) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b3a23f84058..628bd49d1b50 100644 --- a/pom.xml +++ b/pom.xml @@ -626,7 +626,7 @@ 2.2.1 1.0.58 2.12.3 - 1.78 + 1.81 1.5.1 1.0.1 1.1.0 From 899c4ae1068acfa57de4494f71ee454acbe707e6 Mon Sep 17 00:00:00 2001 From: Chandra Sekhar K Date: Fri, 15 Aug 2025 17:40:54 +0530 Subject: [PATCH 021/336] HBASE-29290 Include port number of Region Server in the Replication Status message (#7212) Signed-off-by: Pankaj Kumar , Peng Lu --- hbase-shell/src/main/ruby/hbase/admin.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index c37dfb2e2485..8904191a63a5 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -936,7 +936,7 @@ def status(format, type) r_load_source_map = sl.getReplicationLoadSourceMap build_source_string(r_load_source_map, r_source_string) - puts(format(' %s:', host: server_status.getHostname)) + puts(format(' %s:%s %s', host: server_status.getHostname, port:server_status.getPort, startcode: server_status.getStartcode)) if type.casecmp('SOURCE').zero? puts(format('%s', source: r_source_string)) elsif type.casecmp('SINK').zero? From cddc947eddb2fab5830eb3926964656040fd8c5c Mon Sep 17 00:00:00 2001 From: SVJH Date: Mon, 9 Dec 2019 14:25:37 +0100 Subject: [PATCH 022/336] HBASE-29528 Support cellVisility in Thrift interface Close #7229 Co-authored-by: Duo Zhang Signed-off-by: Duo Zhang (cherry picked from commit 9a2989cbab5f86cb5a2379b6b60d5f5bfb227653) --- .../hbase/thrift/generated/AlreadyExists.java | 2 +- .../hbase/thrift/generated/BatchMutation.java | 2 +- .../thrift/generated/ColumnDescriptor.java | 2 +- .../hadoop/hbase/thrift/generated/Hbase.java | 2 +- .../hbase/thrift/generated/IOError.java | 2 +- .../thrift/generated/IllegalArgument.java | 2 +- .../hbase/thrift/generated/Mutation.java | 2 +- .../hbase/thrift/generated/TAppend.java | 2 +- .../hadoop/hbase/thrift/generated/TCell.java | 2 +- .../hbase/thrift/generated/TColumn.java | 2 +- .../hbase/thrift/generated/TIncrement.java | 2 +- .../hbase/thrift/generated/TRegionInfo.java | 2 +- .../hbase/thrift/generated/TRowResult.java | 2 +- .../hadoop/hbase/thrift/generated/TScan.java | 2 +- .../thrift/generated/TThriftServerType.java | 2 +- .../hadoop/hbase/thrift2/ThriftUtilities.java | 4 + .../hbase/thrift2/generated/TAppend.java | 2 +- .../thrift2/generated/TAuthorization.java | 2 +- .../thrift2/generated/TBloomFilterType.java | 2 +- .../thrift2/generated/TCellVisibility.java | 2 +- .../hbase/thrift2/generated/TColumn.java | 2 +- .../generated/TColumnFamilyDescriptor.java | 2 +- .../thrift2/generated/TColumnIncrement.java | 2 +- .../hbase/thrift2/generated/TColumnValue.java | 2 +- .../hbase/thrift2/generated/TCompareOp.java | 2 +- .../generated/TCompressionAlgorithm.java | 2 +- .../hbase/thrift2/generated/TConsistency.java | 2 +- .../thrift2/generated/TDataBlockEncoding.java | 2 +- .../hbase/thrift2/generated/TDelete.java | 122 +++++++++++++++++- .../hbase/thrift2/generated/TDeleteType.java | 2 +- .../hbase/thrift2/generated/TDurability.java | 2 +- .../thrift2/generated/TFilterByOperator.java | 2 +- .../hadoop/hbase/thrift2/generated/TGet.java | 2 +- .../thrift2/generated/THBaseService.java | 2 +- .../hbase/thrift2/generated/THRegionInfo.java | 2 +- .../thrift2/generated/THRegionLocation.java | 2 +- .../hbase/thrift2/generated/TIOError.java | 2 +- .../thrift2/generated/TIllegalArgument.java | 2 +- .../hbase/thrift2/generated/TIncrement.java | 2 +- .../thrift2/generated/TKeepDeletedCells.java | 2 +- .../thrift2/generated/TLogQueryFilter.java | 2 +- .../hbase/thrift2/generated/TLogType.java | 2 +- .../hbase/thrift2/generated/TMutation.java | 2 +- .../generated/TNamespaceDescriptor.java | 2 +- .../thrift2/generated/TOnlineLogRecord.java | 2 +- .../hadoop/hbase/thrift2/generated/TPut.java | 2 +- .../hbase/thrift2/generated/TReadType.java | 2 +- .../hbase/thrift2/generated/TResult.java | 2 +- .../thrift2/generated/TRowMutations.java | 2 +- .../hadoop/hbase/thrift2/generated/TScan.java | 2 +- .../hbase/thrift2/generated/TServerName.java | 2 +- .../thrift2/generated/TTableDescriptor.java | 2 +- .../hbase/thrift2/generated/TTableName.java | 2 +- .../thrift2/generated/TThriftServerType.java | 2 +- .../hbase/thrift2/generated/TTimeRange.java | 2 +- .../apache/hadoop/hbase/thrift2/hbase.thrift | 3 +- ...stThriftHBaseServiceHandlerWithLabels.java | 89 +++++++++++++ 57 files changed, 265 insertions(+), 59 deletions(-) diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java index 74e7c8143aa5..0feba517601d 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/AlreadyExists.java @@ -11,7 +11,7 @@ * An AlreadyExists exceptions signals that a table with the specified * name already exists */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class AlreadyExists extends org.apache.thrift.TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("AlreadyExists"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java index 6dfab3266f53..6e82601fec85 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/BatchMutation.java @@ -10,7 +10,7 @@ /** * A BatchMutation object is used to apply a number of Mutations to a single row. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class BatchMutation implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("BatchMutation"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java index cf52018bdd9b..e49880f4cebf 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java @@ -12,7 +12,7 @@ * such as the number of versions, compression settings, etc. It is * used as input when creating a table or adding a column. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class ColumnDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnDescriptor"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java index 4ba89955c3cf..279ff8afc99e 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Hbase.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class Hbase { public interface Iface { diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java index d9e913ca022f..67691335f04c 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IOError.java @@ -12,7 +12,7 @@ * to the Hbase master or an Hbase region server. Also used to return * more general Hbase error conditions. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class IOError extends org.apache.thrift.TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IOError"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java index e0efed832bbc..fbb6f2a0524d 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/IllegalArgument.java @@ -11,7 +11,7 @@ * An IllegalArgument exception indicates an illegal or invalid * argument was passed into a procedure. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class IllegalArgument extends org.apache.thrift.TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IllegalArgument"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java index c1c38b06540d..138b8c2171e9 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/Mutation.java @@ -10,7 +10,7 @@ /** * A Mutation object is used to either update or delete a column-value. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class Mutation implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Mutation"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TAppend.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TAppend.java index 8fff5adaa88c..67874e6652f0 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TAppend.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TAppend.java @@ -10,7 +10,7 @@ /** * An Append object is used to specify the parameters for performing the append operation. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TAppend implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TAppend"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java index c22cd2691d62..fab301ea37de 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TCell.java @@ -13,7 +13,7 @@ * the timestamp of a cell to a first-class value, making it easy to take * note of temporal data. Cell is used all the way from HStore up to HTable. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TCell implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCell"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java index 8932017d16c4..3741a337b5ed 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TColumn.java @@ -10,7 +10,7 @@ /** * Holds column name and the cell. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TColumn implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java index a5e75942f933..0d8fb2d0db0c 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TIncrement.java @@ -11,7 +11,7 @@ * For increments that are not incrementColumnValue * equivalents. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIncrement"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java index 2fcd55435f6d..7092af2f368b 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRegionInfo.java @@ -10,7 +10,7 @@ /** * A TRegionInfo contains information about an HTable region. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TRegionInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRegionInfo"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java index a5729c681447..f17cc4acb73e 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TRowResult.java @@ -10,7 +10,7 @@ /** * Holds row name and then a map of columns to cells. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TRowResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowResult"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java index 993e85762e82..4293e9ce47b6 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TScan.java @@ -10,7 +10,7 @@ /** * A Scan object is used to specify scanner parameters when opening a scanner. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TScan implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TThriftServerType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TThriftServerType.java index dbe670ab39f1..0c7d3bcd35db 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TThriftServerType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/generated/TThriftServerType.java @@ -10,7 +10,7 @@ /** * Specify type of thrift server: thrift and thrift2 */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TThriftServerType implements org.apache.thrift.TEnum { ONE(1), TWO(2); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index b3fbaff828a5..4694d82e87a2 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -384,6 +384,10 @@ public static Delete deleteFromThrift(TDelete in) { out.setDurability(durabilityFromThrift(in.getDurability())); } + if (in.getCellVisibility() != null) { + out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression())); + } + return out; } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAppend.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAppend.java index 67085a70eb56..940be40dbedf 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAppend.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAppend.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TAppend implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TAppend"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAuthorization.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAuthorization.java index 3060de04557d..d2ef45942688 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAuthorization.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TAuthorization.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TAuthorization implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TAuthorization"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TBloomFilterType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TBloomFilterType.java index e999712619ba..1ffed27cfd70 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TBloomFilterType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TBloomFilterType.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.regionserver.BloomType */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TBloomFilterType implements org.apache.thrift.TEnum { /** * Bloomfilters disabled diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCellVisibility.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCellVisibility.java index 7d8f7d27e6b7..0451ff8ddba3 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCellVisibility.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCellVisibility.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TCellVisibility implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TCellVisibility"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java index f7706731e61d..a85d5b2b92ac 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumn.java @@ -12,7 +12,7 @@ * in a HBase table by column family and optionally * a column qualifier and timestamp */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TColumn implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumn"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnFamilyDescriptor.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnFamilyDescriptor.java index 5d557d164d1c..f7760d11a3ca 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnFamilyDescriptor.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnFamilyDescriptor.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.client.ColumnFamilyDescriptor */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TColumnFamilyDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnFamilyDescriptor"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java index 1321962f7991..efcc839ffe34 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnIncrement.java @@ -10,7 +10,7 @@ /** * Represents a single cell and the amount to increment it by */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TColumnIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnIncrement"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java index 945cd19bf765..cf06c4302df9 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TColumnValue.java @@ -10,7 +10,7 @@ /** * Represents a single cell and its value. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TColumnValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TColumnValue"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompareOp.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompareOp.java index 2e3ca1939eea..5d45980d87b3 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompareOp.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompareOp.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.filter.CompareFilter$CompareOp. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TCompareOp implements org.apache.thrift.TEnum { LESS(0), LESS_OR_EQUAL(1), diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompressionAlgorithm.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompressionAlgorithm.java index d68bdb870375..2ddd74cb7895 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompressionAlgorithm.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TCompressionAlgorithm.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.io.compress.Algorithm */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TCompressionAlgorithm implements org.apache.thrift.TEnum { LZO(0), GZ(1), diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TConsistency.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TConsistency.java index 8794ce8821ac..49b17381728b 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TConsistency.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TConsistency.java @@ -12,7 +12,7 @@ * - STRONG means reads only from primary region * - TIMELINE means reads might return values from secondary region replicas */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TConsistency implements org.apache.thrift.TEnum { STRONG(1), TIMELINE(2); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDataBlockEncoding.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDataBlockEncoding.java index d8a325e78a5f..2b9f5f16fb6b 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDataBlockEncoding.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDataBlockEncoding.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.io.encoding.DataBlockEncoding */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TDataBlockEncoding implements org.apache.thrift.TEnum { /** * Disable data block encoding. diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java index a7f682f9ee2b..92fa3c7ae69c 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDelete.java @@ -33,7 +33,7 @@ * by changing the durability. If you don't provide durability, it defaults to * column family's default setting for durability. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TDelete implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDelete"); @@ -43,6 +43,7 @@ public class TDelete implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -105,6 +108,8 @@ public static _Fields findByThriftId(int fieldId) { return ATTRIBUTES; case 7: // DURABILITY return DURABILITY; + case 8: // CELL_VISIBILITY + return CELL_VISIBILITY; default: return null; } @@ -148,7 +153,7 @@ public java.lang.String getFieldName() { // isset id assignments private static final int __TIMESTAMP_ISSET_ID = 0; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.ATTRIBUTES,_Fields.DURABILITY}; + private static final _Fields optionals[] = {_Fields.COLUMNS,_Fields.TIMESTAMP,_Fields.DELETE_TYPE,_Fields.ATTRIBUTES,_Fields.DURABILITY,_Fields.CELL_VISIBILITY}; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -167,6 +172,8 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)))); tmpMap.put(_Fields.DURABILITY, new org.apache.thrift.meta_data.FieldMetaData("durability", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TDurability.class))); + tmpMap.put(_Fields.CELL_VISIBILITY, new org.apache.thrift.meta_data.FieldMetaData("cellVisibility", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TCellVisibility.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDelete.class, metaDataMap); } @@ -209,6 +216,9 @@ public TDelete(TDelete other) { if (other.isSetDurability()) { this.durability = other.durability; } + if (other.isSetCellVisibility()) { + this.cellVisibility = new TCellVisibility(other.cellVisibility); + } } public TDelete deepCopy() { @@ -225,6 +235,7 @@ public void clear() { this.attributes = null; this.durability = null; + this.cellVisibility = null; } public byte[] getRow() { @@ -427,6 +438,31 @@ public void setDurabilityIsSet(boolean value) { } } + @org.apache.thrift.annotation.Nullable + public TCellVisibility getCellVisibility() { + return this.cellVisibility; + } + + public TDelete setCellVisibility(@org.apache.thrift.annotation.Nullable TCellVisibility cellVisibility) { + this.cellVisibility = cellVisibility; + return this; + } + + public void unsetCellVisibility() { + this.cellVisibility = null; + } + + /** Returns true if field cellVisibility is set (has been assigned a value) and false otherwise */ + public boolean isSetCellVisibility() { + return this.cellVisibility != null; + } + + public void setCellVisibilityIsSet(boolean value) { + if (!value) { + this.cellVisibility = null; + } + } + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { case ROW: @@ -481,6 +517,14 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; + case CELL_VISIBILITY: + if (value == null) { + unsetCellVisibility(); + } else { + setCellVisibility((TCellVisibility)value); + } + break; + } } @@ -505,6 +549,9 @@ public java.lang.Object getFieldValue(_Fields field) { case DURABILITY: return getDurability(); + case CELL_VISIBILITY: + return getCellVisibility(); + } throw new java.lang.IllegalStateException(); } @@ -528,6 +575,8 @@ public boolean isSet(_Fields field) { return isSetAttributes(); case DURABILITY: return isSetDurability(); + case CELL_VISIBILITY: + return isSetCellVisibility(); } throw new java.lang.IllegalStateException(); } @@ -599,6 +648,15 @@ public boolean equals(TDelete that) { return false; } + boolean this_present_cellVisibility = true && this.isSetCellVisibility(); + boolean that_present_cellVisibility = true && that.isSetCellVisibility(); + if (this_present_cellVisibility || that_present_cellVisibility) { + if (!(this_present_cellVisibility && that_present_cellVisibility)) + return false; + if (!this.cellVisibility.equals(that.cellVisibility)) + return false; + } + return true; } @@ -630,6 +688,10 @@ public int hashCode() { if (isSetDurability()) hashCode = hashCode * 8191 + durability.getValue(); + hashCode = hashCode * 8191 + ((isSetCellVisibility()) ? 131071 : 524287); + if (isSetCellVisibility()) + hashCode = hashCode * 8191 + cellVisibility.hashCode(); + return hashCode; } @@ -701,6 +763,16 @@ public int compareTo(TDelete other) { return lastComparison; } } + lastComparison = java.lang.Boolean.compare(isSetCellVisibility(), other.isSetCellVisibility()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCellVisibility()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.cellVisibility, other.cellVisibility); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -775,6 +847,16 @@ public java.lang.String toString() { } first = false; } + if (isSetCellVisibility()) { + if (!first) sb.append(", "); + sb.append("cellVisibility:"); + if (this.cellVisibility == null) { + sb.append("null"); + } else { + sb.append(this.cellVisibility); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -785,6 +867,9 @@ public void validate() throws org.apache.thrift.TException { throw new org.apache.thrift.protocol.TProtocolException("Required field 'row' was not present! Struct: " + toString()); } // check for sub-struct validity + if (cellVisibility != null) { + cellVisibility.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -894,6 +979,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TDelete struct) thr org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 8: // CELL_VISIBILITY + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.cellVisibility = new TCellVisibility(); + struct.cellVisibility.read(iprot); + struct.setCellVisibilityIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -962,6 +1056,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TDelete struct) th oprot.writeFieldEnd(); } } + if (struct.cellVisibility != null) { + if (struct.isSetCellVisibility()) { + oprot.writeFieldBegin(CELL_VISIBILITY_FIELD_DESC); + struct.cellVisibility.write(oprot); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -996,7 +1097,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr if (struct.isSetDurability()) { optionals.set(4); } - oprot.writeBitSet(optionals, 5); + if (struct.isSetCellVisibility()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); if (struct.isSetColumns()) { { oprot.writeI32(struct.columns.size()); @@ -1025,6 +1129,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thr if (struct.isSetDurability()) { oprot.writeI32(struct.durability.getValue()); } + if (struct.isSetCellVisibility()) { + struct.cellVisibility.write(oprot); + } } @Override @@ -1032,7 +1139,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; struct.row = iprot.readBinary(); struct.setRowIsSet(true); - java.util.BitSet incoming = iprot.readBitSet(5); + java.util.BitSet incoming = iprot.readBitSet(6); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list63 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); @@ -1074,6 +1181,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TDelete struct) thro struct.durability = org.apache.hadoop.hbase.thrift2.generated.TDurability.findByValue(iprot.readI32()); struct.setDurabilityIsSet(true); } + if (incoming.get(5)) { + struct.cellVisibility = new TCellVisibility(); + struct.cellVisibility.read(iprot); + struct.setCellVisibilityIsSet(true); + } } } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java index b25787a68b2a..015436c66075 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDeleteType.java @@ -12,7 +12,7 @@ * - DELETE_COLUMN means exactly one version will be removed, * - DELETE_COLUMNS means previous versions will also be removed. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TDeleteType implements org.apache.thrift.TEnum { DELETE_COLUMN(0), DELETE_COLUMNS(1), diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java index d0d9ebcd353a..97d2a8bc33b7 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TDurability.java @@ -14,7 +14,7 @@ * - SYNC_WAL means write the Mutation to the WAL synchronously, * - FSYNC_WAL means Write the Mutation to the WAL synchronously and force the entries to disk. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TDurability implements org.apache.thrift.TEnum { USE_DEFAULT(0), SKIP_WAL(1), diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TFilterByOperator.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TFilterByOperator.java index b3f975989a82..312dd485a450 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TFilterByOperator.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TFilterByOperator.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TFilterByOperator implements org.apache.thrift.TEnum { AND(0), OR(1); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java index 36e0a2e2d12d..7f200f5ff022 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TGet.java @@ -20,7 +20,7 @@ * If you specify a time range and a timestamp the range is ignored. * Timestamps on TColumns are ignored. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TGet implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TGet"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java index 6015a3b525b6..eb370d7587b1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THBaseService.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class THBaseService { public interface Iface { diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionInfo.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionInfo.java index 87af97f7abd8..1ba7d4f48b52 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionInfo.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionInfo.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class THRegionInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("THRegionInfo"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionLocation.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionLocation.java index e0710477ff8d..208cd424358f 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionLocation.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/THRegionLocation.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class THRegionLocation implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("THRegionLocation"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java index 6f4d925eb4f3..22e807ed30d2 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIOError.java @@ -12,7 +12,7 @@ * to the HBase master or a HBase region server. Also used to return * more general HBase error conditions. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TIOError extends org.apache.thrift.TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIOError"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java index 3fe63b2e5be5..29446f754c88 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIllegalArgument.java @@ -11,7 +11,7 @@ * A TIllegalArgument exception indicates an illegal or invalid * argument was passed into a procedure. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TIllegalArgument extends org.apache.thrift.TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIllegalArgument"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java index 830e0362f44b..ec600aef0c91 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TIncrement.java @@ -14,7 +14,7 @@ * by changing the durability. If you don't provide durability, it defaults to * column family's default setting for durability. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TIncrement implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TIncrement"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TKeepDeletedCells.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TKeepDeletedCells.java index e086de356916..e538e5c46ae0 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TKeepDeletedCells.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TKeepDeletedCells.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.KeepDeletedCells */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TKeepDeletedCells implements org.apache.thrift.TEnum { /** * Deleted Cells are not retained. diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogQueryFilter.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogQueryFilter.java index ebca32395969..0f627742b4eb 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogQueryFilter.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogQueryFilter.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.client.LogQueryFilter */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TLogQueryFilter implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TLogQueryFilter"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogType.java index 48012cc7fbfc..6786a9272b5c 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TLogType.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TLogType implements org.apache.thrift.TEnum { SLOW_LOG(1), LARGE_LOG(2); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java index 523203e5caab..ff4f6616e813 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TMutation.java @@ -10,7 +10,7 @@ /** * Atomic mutation for the specified row. It can be either Put or Delete. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TMutation extends org.apache.thrift.TUnion { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TMutation"); private static final org.apache.thrift.protocol.TField PUT_FIELD_DESC = new org.apache.thrift.protocol.TField("put", org.apache.thrift.protocol.TType.STRUCT, (short)1); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TNamespaceDescriptor.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TNamespaceDescriptor.java index cbce4a084a21..5955eb3e6bf1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TNamespaceDescriptor.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TNamespaceDescriptor.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.NamespaceDescriptor */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TNamespaceDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TNamespaceDescriptor"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java index c3d6cba7ad20..fd6d8b6a4e76 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TOnlineLogRecord.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.client.OnlineLogRecord */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2024-01-12") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TOnlineLogRecord implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TOnlineLogRecord"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java index 08614d63627a..a1bd2fa024b1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TPut.java @@ -19,7 +19,7 @@ * by changing the durability. If you don't provide durability, it defaults to * column family's default setting for durability. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TPut implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TPut"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java index 612aed531db6..82fcfaf6cd95 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TReadType implements org.apache.thrift.TEnum { DEFAULT(1), STREAM(2), diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java index 1cb1a5eac69d..9b0cf1169b02 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TResult.java @@ -10,7 +10,7 @@ /** * if no Result is found, row and columnValues will not be set. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TResult"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java index bc9d2b34fa96..0934e40b4715 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TRowMutations.java @@ -10,7 +10,7 @@ /** * A TRowMutations object is used to apply a number of Mutations to a single row. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TRowMutations implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TRowMutations"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java index 437a6510fedd..cdc8b9774300 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java @@ -11,7 +11,7 @@ * Any timestamps in the columns are ignored but the colFamTimeRangeMap included, use timeRange to select by timestamp. * Max versions defaults to 1. */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TScan implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TServerName.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TServerName.java index 841ffa63f934..13fe1ce05156 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TServerName.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TServerName.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TServerName implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TServerName"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableDescriptor.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableDescriptor.java index 8a905481ec0e..c6775dd1ac8f 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableDescriptor.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableDescriptor.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.client.TableDescriptor */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TTableDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTableDescriptor"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableName.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableName.java index 204a96302047..32e8639b192d 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableName.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTableName.java @@ -11,7 +11,7 @@ * Thrift wrapper around * org.apache.hadoop.hbase.TableName */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TTableName implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTableName"); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TThriftServerType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TThriftServerType.java index d0b5c346089f..208bc4a5e1cd 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TThriftServerType.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TThriftServerType.java @@ -10,7 +10,7 @@ /** * Specify type of thrift server: thrift and thrift2 */ -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public enum TThriftServerType implements org.apache.thrift.TEnum { ONE(1), TWO(2); diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java index 81fc51dd0410..7868c705358b 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TTimeRange.java @@ -7,7 +7,7 @@ package org.apache.hadoop.hbase.thrift2.generated; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2022-07-05") +@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.14.1)", date = "2025-08-18") public class TTimeRange implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTimeRange"); diff --git a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift index 675eb237ec3e..af8c045ec1b4 100644 --- a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift +++ b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift @@ -203,7 +203,8 @@ struct TDelete { 3: optional i64 timestamp, 4: optional TDeleteType deleteType = 1, 6: optional map attributes, - 7: optional TDurability durability + 7: optional TDurability durability, + 8: optional TCellVisibility cellVisibility } diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java index 505295d77008..c224e4e2c014 100644 --- a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java +++ b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.thrift2.generated.TColumn; import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement; import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; +import org.apache.hadoop.hbase.thrift2.generated.TDelete; import org.apache.hadoop.hbase.thrift2.generated.TGet; import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument; import org.apache.hadoop.hbase.thrift2.generated.TIncrement; @@ -438,6 +439,94 @@ public void testAppend() throws Exception { assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue()); } + @Test + public void testDeleteWithLabels() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testPutGetDeleteGet".getBytes(); + ByteBuffer table = wrap(tableAname); + + // common auths + TAuthorization tauth = new TAuthorization(); + List labels = new ArrayList(); + labels.add(SECRET); + labels.add(PRIVATE); + tauth.setLabels(labels); + + // put + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); + columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); + TPut put = new TPut(wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + put.setCellVisibility(new TCellVisibility() + .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); + handler.put(table, put); + + // verify put + TGet get = new TGet(wrap(rowName)); + get.setAuthorizations(tauth); + TResult result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + List returnedColumnValues = result.getColumnValues(); + assertTColumnValuesEqual(columnValues, returnedColumnValues); + + // delete + TDelete delete = new TDelete(wrap(rowName)); + delete.setCellVisibility(new TCellVisibility() + .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); + handler.deleteSingle(table, delete); + + // verify delete + TGet get2 = new TGet(wrap(rowName)); + get2.setAuthorizations(tauth); + TResult result2 = handler.get(table, get2); + assertNull(result2.getRow()); + } + + @Test + public void testDeleteWithLabelsNegativeTest() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + byte[] rowName = "testPutGetTryDeleteGet".getBytes(); + ByteBuffer table = wrap(tableAname); + + // common auths + TAuthorization tauth = new TAuthorization(); + List labels = new ArrayList(); + labels.add(SECRET); + labels.add(PRIVATE); + tauth.setLabels(labels); + + // put + List columnValues = new ArrayList(); + columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); + columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); + TPut put = new TPut(wrap(rowName), columnValues); + + put.setColumnValues(columnValues); + put.setCellVisibility(new TCellVisibility() + .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); + handler.put(table, put); + + // verify put + TGet get = new TGet(wrap(rowName)); + get.setAuthorizations(tauth); + TResult result = handler.get(table, get); + assertArrayEquals(rowName, result.getRow()); + List returnedColumnValues = result.getColumnValues(); + assertTColumnValuesEqual(columnValues, returnedColumnValues); + + // _try_ delete with _no_ CellVisibility + TDelete delete = new TDelete(wrap(rowName)); + handler.deleteSingle(table, delete); + + // verify delete did in fact _not_ work + TGet get2 = new TGet(wrap(rowName)); + get2.setAuthorizations(tauth); + TResult result2 = handler.get(table, get2); + assertArrayEquals(rowName, result2.getRow()); + } + /** * Padding numbers to make comparison of sort order easier in a for loop The number to pad. * @param n The number to pad. From 22ba3bf98d132e828c66d9247091e6472597eaa8 Mon Sep 17 00:00:00 2001 From: mokai Date: Tue, 19 Aug 2025 00:06:33 +0800 Subject: [PATCH 023/336] HBASE-29473 Obtain target cluster's token for cross clusters job (#7198) Signed-off-by: Nihal Jain Signed-off-by: Junegunn Choi Signed-off-by: Pankaj Kumar Reviewed-by: chaijunjie <1340011734@qq.com> (cherry picked from commit 4307f21bcbc30fd7c36460d3fb81b575a122a41e) --- .../hbase/mapreduce/HFileOutputFormat2.java | 4 +- .../TestHFileOutputFormat2WithSecurity.java | 132 ++++++++++++++++++ .../mapreduce/TestTableMapReduceUtil.java | 46 +----- .../hadoop/hbase/HBaseTestingUtility.java | 44 ++++++ 4 files changed, 186 insertions(+), 40 deletions(-) create mode 100644 hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 6ab3bdd25048..cb2c62601712 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -787,7 +787,7 @@ public static void configureIncrementalLoadMap(Job job, TableDescriptor tableDes * @see #REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY */ - public static void configureRemoteCluster(Job job, Configuration clusterConf) { + public static void configureRemoteCluster(Job job, Configuration clusterConf) throws IOException { Configuration conf = job.getConfiguration(); if (!conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { @@ -804,6 +804,8 @@ public static void configureRemoteCluster(Job job, Configuration clusterConf) { conf.setInt(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY, clientPort); conf.set(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY, parent); + TableMapReduceUtil.initCredentialsForCluster(job, clusterConf); + LOG.info("ZK configs for remote cluster of bulkload is configured: " + quorum + ":" + clientPort + "/" + parent); } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java new file mode 100644 index 000000000000..b4cb6a8355fc --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.Closeable; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for {@link HFileOutputFormat2} with secure mode. + */ +@Category({ VerySlowMapReduceTests.class, LargeTests.class }) +public class TestHFileOutputFormat2WithSecurity { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHFileOutputFormat2WithSecurity.class); + + private static final byte[] FAMILIES = Bytes.toBytes("test_cf"); + + private static final String HTTP_PRINCIPAL = "HTTP/localhost"; + + private HBaseTestingUtility utilA; + + private Configuration confA; + + private HBaseTestingUtility utilB; + + private MiniKdc kdc; + + private List clusters = new ArrayList<>(); + + @Before + public void setupSecurityClusters() throws Exception { + utilA = new HBaseTestingUtility(); + confA = utilA.getConfiguration(); + + utilB = new HBaseTestingUtility(); + + // Prepare security configs. + File keytab = new File(utilA.getDataTestDir("keytab").toUri().getPath()); + kdc = utilA.setupMiniKdc(keytab); + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + // Start security clusterA + clusters.add(utilA.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + + // Start security clusterB + clusters.add(utilB.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + } + + @After + public void teardownSecurityClusters() { + IOUtils.closeQuietly(clusters); + clusters.clear(); + if (kdc != null) { + kdc.stop(); + } + } + + @Test + public void testIncrementalLoadInMultiClusterWithSecurity() throws Exception { + TableName tableName = TableName.valueOf("testIncrementalLoadInMultiClusterWithSecurity"); + + // Create table in clusterB + try (Table table = utilB.createTable(tableName, FAMILIES); + RegionLocator r = utilB.getConnection().getRegionLocator(tableName)) { + + // Create job in clusterA + Job job = Job.getInstance(confA, "testIncrementalLoadInMultiClusterWithSecurity"); + job.setWorkingDirectory( + utilA.getDataTestDirOnTestFS("testIncrementalLoadInMultiClusterWithSecurity")); + job.setInputFormatClass(NMapInputFormat.class); + job.setMapperClass(TestHFileOutputFormat2.RandomKVGeneratingMapper.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(KeyValue.class); + HFileOutputFormat2.configureIncrementalLoad(job, table, r); + + assertEquals(2, job.getCredentials().getAllTokens().size()); + + String remoteClusterId = utilB.getHBaseClusterInterface().getClusterMetrics().getClusterId(); + assertTrue(job.getCredentials().getToken(new Text(remoteClusterId)) != null); + } finally { + if (utilB.getAdmin().tableExists(tableName)) { + utilB.deleteTable(tableName); + } + } + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index 3b7392b3ae45..f661025ac062 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -29,15 +29,8 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.security.HBaseKerberosUtils; -import org.apache.hadoop.hbase.security.access.AccessController; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; -import org.apache.hadoop.hbase.security.token.TokenProvider; -import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -48,7 +41,6 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.junit.After; @@ -134,33 +126,6 @@ public void testInitTableMapperJob4() throws Exception { assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } - private static Closeable startSecureMiniCluster(HBaseTestingUtility util, MiniKdc kdc, - String principal) throws Exception { - Configuration conf = util.getConfiguration(); - - SecureTestUtil.enableSecurity(conf); - VisibilityTestUtil.enableVisiblityLabels(conf); - SecureTestUtil.verifyConfiguration(conf); - - conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - AccessController.class.getName() + ',' + TokenProvider.class.getName()); - - HBaseKerberosUtils.setSecuredConfiguration(conf, principal + '@' + kdc.getRealm(), - HTTP_PRINCIPAL + '@' + kdc.getRealm()); - - KerberosName.resetDefaultRealm(); - - util.startMiniCluster(); - try { - util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); - } catch (Exception e) { - util.shutdownMiniCluster(); - throw e; - } - - return util::shutdownMiniCluster; - } - @Test public void testInitCredentialsForCluster1() throws Exception { HBaseTestingUtility util1 = new HBaseTestingUtility(); @@ -199,8 +164,9 @@ public void testInitCredentialsForCluster2() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal); - Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL); + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); @@ -233,7 +199,8 @@ public void testInitCredentialsForCluster3() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { HBaseTestingUtility util2 = new HBaseTestingUtility(); // Assume util2 is insecure cluster @@ -269,7 +236,8 @@ public void testInitCredentialsForCluster4() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 01a9c4f06fb7..3db73e1ad5f2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -89,6 +90,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; @@ -121,7 +123,12 @@ import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; +import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -152,6 +159,7 @@ import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.ZooKeeper; @@ -394,6 +402,42 @@ public static void closeRegionAndWAL(final HRegion r) throws IOException { r.getWAL().close(); } + /** + * Start mini secure cluster with given kdc and principals. + * @param kdc Mini kdc server + * @param servicePrincipal Service principal without realm. + * @param spnegoPrincipal Spnego principal without realm. + * @return Handler to shutdown the cluster + */ + public Closeable startSecureMiniCluster(MiniKdc kdc, String servicePrincipal, + String spnegoPrincipal) throws Exception { + Configuration conf = getConfiguration(); + + SecureTestUtil.enableSecurity(conf); + VisibilityTestUtil.enableVisiblityLabels(conf); + SecureTestUtil.verifyConfiguration(conf); + + // Reset the static default realm forcibly for hadoop-2.0. + // It has no impact but not required for hadoop-3.0. + KerberosName.resetDefaultRealm(); + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + AccessController.class.getName() + ',' + TokenProvider.class.getName()); + + HBaseKerberosUtils.setSecuredConfiguration(conf, servicePrincipal + '@' + kdc.getRealm(), + spnegoPrincipal + '@' + kdc.getRealm()); + + startMiniCluster(); + try { + waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + } catch (Exception e) { + shutdownMiniCluster(); + throw e; + } + + return this::shutdownMiniCluster; + } + /** * Returns this classes's instance of {@link Configuration}. Be careful how you use the returned * Configuration since {@link Connection} instances can be shared. The Map of Connections is keyed From 789a8d852e67c85365514ff36850dfc9250de0d0 Mon Sep 17 00:00:00 2001 From: Umesh <9414umeshkumar@gmail.com> Date: Wed, 20 Aug 2025 07:40:47 +0530 Subject: [PATCH 024/336] HBASE-28951 Handle simultaneous WAL splitting to recovered edits by multiple worker (#7228) (#7075) Signed-off-by: Andrew Purtell Signed-off-by: Duo Zhang Signed-off-by: Aman Poonia --- .../wal/AbstractRecoveredEditsOutputSink.java | 102 +++++++++++++----- .../apache/hadoop/hbase/wal/WALSplitUtil.java | 19 ++-- .../apache/hadoop/hbase/wal/TestWALSplit.java | 69 +++++++++++- 3 files changed, 157 insertions(+), 33 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractRecoveredEditsOutputSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractRecoveredEditsOutputSink.java index 89edccc22538..d447e9ce6b8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractRecoveredEditsOutputSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractRecoveredEditsOutputSink.java @@ -22,6 +22,9 @@ import java.io.EOFException; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -30,8 +33,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.log.HBaseMarkers; +import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; @@ -45,6 +50,7 @@ abstract class AbstractRecoveredEditsOutputSink extends OutputSink { private static final Logger LOG = LoggerFactory.getLogger(RecoveredEditsOutputSink.class); private final WALSplitter walSplitter; private final ConcurrentMap regionMaximumEditLogSeqNum = new ConcurrentHashMap<>(); + private static final int MAX_RENAME_RETRY_COUNT = 5; public AbstractRecoveredEditsOutputSink(WALSplitter walSplitter, WALSplitter.PipelineController controller, EntryBuffers entryBuffers, int numWriters) { @@ -55,9 +61,12 @@ public AbstractRecoveredEditsOutputSink(WALSplitter walSplitter, /** Returns a writer that wraps a {@link WALProvider.Writer} and its Path. Caller should close. */ protected RecoveredEditsWriter createRecoveredEditsWriter(TableName tableName, byte[] region, long seqId) throws IOException { + // If multiple worker are splitting a WAL at a same time, both should use unique file name to + // avoid conflict Path regionEditsPath = getRegionSplitEditsPath(tableName, region, seqId, walSplitter.getFileBeingSplit().getPath().getName(), walSplitter.getTmpDirName(), - walSplitter.conf); + walSplitter.conf, getWorkerNameComponent()); + if (walSplitter.walFS.exists(regionEditsPath)) { LOG.warn("Found old edits file. It could be the " + "result of a previous failed split attempt. Deleting " + regionEditsPath + ", length=" @@ -73,6 +82,20 @@ protected RecoveredEditsWriter createRecoveredEditsWriter(TableName tableName, b return new RecoveredEditsWriter(region, regionEditsPath, w, seqId); } + private String getWorkerNameComponent() { + if (walSplitter.rsServices == null) { + return ""; + } + try { + return URLEncoder.encode( + walSplitter.rsServices.getServerName().toShortString() + .replace(Addressing.HOSTNAME_PORT_SEPARATOR, ServerName.SERVERNAME_SEPARATOR), + StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("URLEncoder doesn't support UTF-8", e); + } + } + /** * abortRecoveredEditsWriter closes the editsWriter, but does not rename and finalize the * recovered edits WAL files. Please see HBASE-28569. @@ -103,22 +126,40 @@ protected Path closeRecoveredEditsWriterAndFinalizeEdits(RecoveredEditsWriter ed Path dst = getCompletedRecoveredEditsFilePath(editsWriter.path, regionMaximumEditLogSeqNum.get(Bytes.toString(editsWriter.encodedRegionName))); try { - if (!dst.equals(editsWriter.path) && walSplitter.walFS.exists(dst)) { - deleteOneWithFewerEntries(editsWriter, dst); - } // Skip the unit tests which create a splitter that reads and // writes the data without touching disk. // TestHLogSplit#testThreading is an example. if (walSplitter.walFS.exists(editsWriter.path)) { - if (!walSplitter.walFS.rename(editsWriter.path, dst)) { - final String errorMsg = - "Failed renaming recovered edits " + editsWriter.path + " to " + dst; + boolean retry; + int retryCount = 0; + do { + retry = false; + retryCount++; + // If rename is successful, it means recovered edits are successfully places at right + // place but if rename fails, there can be below reasons + // 1. dst already exist - in this case if dst have desired edits we will keep the dst, + // delete the editsWriter.path and consider this success else if dst have fewer edits, we + // will delete the dst and retry the rename + // 2. parent directory does not exit - in one edge case this is possible when this worker + // got stuck before rename and HMaster get another worker to split the wal, SCP will + // proceed and once region get opened on one RS, we delete the recovered.edits directory, + // in this case there is no harm in failing this procedure after retry exhausted. + if (!walSplitter.walFS.rename(editsWriter.path, dst)) { + retry = deleteOneWithFewerEntriesToRetry(editsWriter, dst); + } + } while (retry && retryCount < MAX_RENAME_RETRY_COUNT); + + // If we are out of loop with retry flag `true` it means we have exhausted the retries. + if (retry) { + final String errorMsg = "Failed renaming recovered edits " + editsWriter.path + " to " + + dst + " in " + MAX_RENAME_RETRY_COUNT + " retries"; updateStatusWithMsg(errorMsg); throw new IOException(errorMsg); + } else { + final String renameEditMsg = "Rename recovered edits " + editsWriter.path + " to " + dst; + LOG.info(renameEditMsg); + updateStatusWithMsg(renameEditMsg); } - final String renameEditMsg = "Rename recovered edits " + editsWriter.path + " to " + dst; - LOG.info(renameEditMsg); - updateStatusWithMsg(renameEditMsg); } } catch (IOException ioe) { final String errorMsg = "Could not rename recovered edits " + editsWriter.path + " to " + dst; @@ -186,36 +227,49 @@ void updateRegionMaximumEditLogSeqNum(WAL.Entry entry) { } // delete the one with fewer wal entries - private void deleteOneWithFewerEntries(RecoveredEditsWriter editsWriter, Path dst) + private boolean deleteOneWithFewerEntriesToRetry(RecoveredEditsWriter editsWriter, Path dst) throws IOException { - long dstMinLogSeqNum = -1L; - try (WALStreamReader reader = - walSplitter.getWalFactory().createStreamReader(walSplitter.walFS, dst)) { - WAL.Entry entry = reader.next(); - if (entry != null) { - dstMinLogSeqNum = entry.getKey().getSequenceId(); - } - } catch (EOFException e) { - LOG.debug("Got EOF when reading first WAL entry from {}, an empty or broken WAL file?", dst, - e); + if (!walSplitter.walFS.exists(dst)) { + LOG.info("dst {} doesn't exist, need to retry ", dst); + return true; } - if (editsWriter.minLogSeqNum < dstMinLogSeqNum) { + + if (isDstHasFewerEntries(editsWriter, dst)) { LOG.warn("Found existing old edits file. It could be the result of a previous failed" + " split attempt or we have duplicated wal entries. Deleting " + dst + ", length=" - + walSplitter.walFS.getFileStatus(dst).getLen()); + + walSplitter.walFS.getFileStatus(dst).getLen() + " and retry is needed"); if (!walSplitter.walFS.delete(dst, false)) { LOG.warn("Failed deleting of old {}", dst); throw new IOException("Failed deleting of old " + dst); } + return true; } else { LOG .warn("Found existing old edits file and we have less entries. Deleting " + editsWriter.path - + ", length=" + walSplitter.walFS.getFileStatus(editsWriter.path).getLen()); + + ", length=" + walSplitter.walFS.getFileStatus(editsWriter.path).getLen() + + " and no retry needed as dst has all edits"); if (!walSplitter.walFS.delete(editsWriter.path, false)) { LOG.warn("Failed deleting of {}", editsWriter.path); throw new IOException("Failed deleting of " + editsWriter.path); } + return false; + } + } + + private boolean isDstHasFewerEntries(RecoveredEditsWriter editsWriter, Path dst) + throws IOException { + long dstMinLogSeqNum = -1L; + try (WALStreamReader reader = + walSplitter.getWalFactory().createStreamReader(walSplitter.walFS, dst)) { + WAL.Entry entry = reader.next(); + if (entry != null) { + dstMinLogSeqNum = entry.getKey().getSequenceId(); + } + } catch (EOFException e) { + LOG.debug("Got EOF when reading first WAL entry from {}, an empty or broken WAL file?", dst, + e); } + return editsWriter.minLogSeqNum < dstMinLogSeqNum; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java index bd9bd6f9cc90..d704caaff12b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java @@ -150,17 +150,19 @@ public static void moveWAL(FileSystem fs, Path p, Path targetDir) throws IOExcep * /hbase/some_table/2323432434/recovered.edits/2332. This method also ensures existence of * RECOVERED_EDITS_DIR under the region creating it if necessary. And also set storage policy for * RECOVERED_EDITS_DIR if WAL_STORAGE_POLICY is configured. - * @param tableName the table name - * @param encodedRegionName the encoded region name - * @param seqId the sequence id which used to generate file name - * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. - * @param tmpDirName of the directory used to sideline old recovered edits file - * @param conf configuration + * @param tableName the table name + * @param encodedRegionName the encoded region name + * @param seqId the sequence id which used to generate file name + * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name. + * @param tmpDirName of the directory used to sideline old recovered edits file + * @param conf configuration + * @param workerNameComponent the worker name component for the file name * @return Path to file into which to dump split log edits. */ @SuppressWarnings("deprecation") static Path getRegionSplitEditsPath(TableName tableName, byte[] encodedRegionName, long seqId, - String fileNameBeingSplit, String tmpDirName, Configuration conf) throws IOException { + String fileNameBeingSplit, String tmpDirName, Configuration conf, String workerNameComponent) + throws IOException { FileSystem walFS = CommonFSUtils.getWALFileSystem(conf); Path tableDir = CommonFSUtils.getWALTableDir(conf, tableName); String encodedRegionNameStr = Bytes.toString(encodedRegionName); @@ -192,7 +194,8 @@ static Path getRegionSplitEditsPath(TableName tableName, byte[] encodedRegionNam // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure // region's replayRecoveredEdits will not delete it String fileName = formatRecoveredEditsFileName(seqId); - fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit); + fileName = + getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit + "-" + workerNameComponent); return new Path(dir, fileName); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index 8f8cd38446f2..176fe845fccd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -66,6 +66,8 @@ import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination; import org.apache.hadoop.hbase.master.SplitLogManager; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.LastSequenceId; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufWALReader; import org.apache.hadoop.hbase.regionserver.wal.FaultyProtobufWALStreamReader; import org.apache.hadoop.hbase.regionserver.wal.InstrumentedLogWriter; @@ -105,6 +107,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; /** @@ -372,6 +375,70 @@ private void loop(final Writer writer) { } } + // If another worker is assigned to split a WAl and last worker is still running, both should not + // impact each other's progress + @Test + public void testTwoWorkerSplittingSameWAL() throws IOException, InterruptedException { + int numWriter = 1, entries = 10; + generateWALs(numWriter, entries, -1, 0); + FileStatus logfile = fs.listStatus(WALDIR)[0]; + FileSystem spiedFs = Mockito.spy(fs); + RegionServerServices zombieRSServices = Mockito.mock(RegionServerServices.class); + RegionServerServices newWorkerRSServices = Mockito.mock(RegionServerServices.class); + Mockito.when(zombieRSServices.getServerName()) + .thenReturn(ServerName.valueOf("zombie-rs.abc.com,1234,1234567890")); + Mockito.when(newWorkerRSServices.getServerName()) + .thenReturn(ServerName.valueOf("worker-rs.abc.com,1234,1234569870")); + Thread zombieWorker = new SplitWALWorker(logfile, spiedFs, zombieRSServices); + Thread newWorker = new SplitWALWorker(logfile, spiedFs, newWorkerRSServices); + zombieWorker.start(); + newWorker.start(); + newWorker.join(); + zombieWorker.join(); + + for (String region : REGIONS) { + Path[] logfiles = getLogForRegion(TABLE_NAME, region); + assertEquals("wrong number of split files for region", numWriter, logfiles.length); + + int count = 0; + for (Path lf : logfiles) { + count += countWAL(lf); + } + assertEquals("wrong number of edits for region " + region, entries, count); + } + } + + private class SplitWALWorker extends Thread implements LastSequenceId { + final FileStatus logfile; + final FileSystem fs; + final RegionServerServices rsServices; + + public SplitWALWorker(FileStatus logfile, FileSystem fs, RegionServerServices rsServices) { + super(rsServices.getServerName().toShortString()); + setDaemon(true); + this.fs = fs; + this.logfile = logfile; + this.rsServices = rsServices; + } + + @Override + public void run() { + try { + boolean ret = + WALSplitter.splitLogFile(HBASEDIR, logfile, fs, conf, null, this, null, wals, rsServices); + assertTrue("Both splitting should pass", ret); + } catch (IOException e) { + LOG.warn(getName() + " Worker exiting " + e); + } + } + + @Override + public ClusterStatusProtos.RegionStoreSequenceIds getLastSequenceId(byte[] encodedRegionName) { + return ClusterStatusProtos.RegionStoreSequenceIds.newBuilder() + .setLastFlushedSequenceId(HConstants.NO_SEQNUM).build(); + } + } + /** * @see "https://issues.apache.org/jira/browse/HBASE-3020" */ @@ -403,7 +470,7 @@ public void testOldRecoveredEditsFileSidelined() throws IOException { private Path createRecoveredEditsPathForRegion() throws IOException { byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); Path p = WALSplitUtil.getRegionSplitEditsPath(TableName.META_TABLE_NAME, encoded, 1, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + FILENAME_BEING_SPLIT, TMPDIRNAME, conf, ""); return p; } From d22282cba31e9b591792ec8ef6aac3a994c442e7 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 21 Aug 2025 15:35:14 +0530 Subject: [PATCH 025/336] HBASE-29509 Bump hbase-thirdparty to 4.1.12 (#7209) (#7208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dávid Paksy Signed-off-by: Duo Zhang Signed-off-by: Istvan Toth (cherry picked from commit b0ddff96a946b08044929079b278000060fe3335) --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 628bd49d1b50..c8f3e0f3272d 100644 --- a/pom.xml +++ b/pom.xml @@ -566,7 +566,7 @@ in the dependencyManagement section as it could still lead to different versions of netty modules and cause trouble if we only rely on transitive dependencies. --> - 4.1.121.Final + 4.1.123.Final 0.13.0 - 2.19.0 - 2.19.0 + 2.19.2 + 2.19.2 2.3.1 3.1.0 2.1.1 @@ -612,7 +612,7 @@ Version of protobuf that hbase uses internally (we shade our pb) Must match what is out in hbase-thirdparty include. --> - 4.30.2 + 4.31.1 0.6.1 thrift 0.14.1 @@ -675,7 +675,7 @@ databind] must be kept in sync with the version of jackson-jaxrs-json-provider shipped in hbase-thirdparty. --> - 4.1.11 + 4.1.12 0.8.8 From 7b83eb4ff5984a49c742c159a00518f64826475d Mon Sep 17 00:00:00 2001 From: Peter Somogyi Date: Mon, 25 Aug 2025 15:45:47 +0200 Subject: [PATCH 026/336] HBASE-29543 Fix TestFileChangeWatcher Java 8 time granularity issue (#7245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andor Molnár (cherry picked from commit ddc31b8f7799447d6f0e04dce7f2a2ca6ebcf90a) --- .../apache/hadoop/hbase/io/TestFileChangeWatcher.java | 9 +++++---- .../hbase/security/TestNettyTLSIPCFileWatcher.java | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java index c24e96f8da6b..484a3de31434 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestFileChangeWatcher.java @@ -125,7 +125,7 @@ public void testNoFalseNotifications() throws IOException, InterruptedException }); watcher.start(); watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) assertEquals("Should not have been notified", 0, notifiedPaths.size()); } finally { if (watcher != null) { @@ -149,8 +149,8 @@ public void testCallbackWorksOnFileChanges() throws IOException, InterruptedExce }); watcher.start(); watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack for (int i = 0; i < 3; i++) { + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) LOG.info("Modifying file, attempt {}", (i + 1)); FileUtils.writeStringToFile(tempFile, "Hello world " + i + "\n", StandardCharsets.UTF_8, true); @@ -185,7 +185,7 @@ public void testCallbackWorksOnFileTouched() throws IOException, InterruptedExce }); watcher.start(); watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) LOG.info("Touching file"); FileUtils.touch(tempFile); synchronized (notifiedPaths) { @@ -223,7 +223,7 @@ public void testCallbackErrorDoesNotCrashWatcherThread() }); watcher.start(); watcher.waitForState(FileChangeWatcher.State.RUNNING); - Thread.sleep(1000L); // TODO hack + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) LOG.info("Modifying file"); FileUtils.writeStringToFile(tempFile, "Hello world\n", StandardCharsets.UTF_8, true); synchronized (callCount) { @@ -231,6 +231,7 @@ public void testCallbackErrorDoesNotCrashWatcherThread() callCount.wait(FS_TIMEOUT); } } + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) LOG.info("Modifying file again"); FileUtils.writeStringToFile(tempFile, "Hello world again\n", StandardCharsets.UTF_8, true); synchronized (callCount) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java index ede88169c3bc..6d9fa5f67514 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestNettyTLSIPCFileWatcher.java @@ -190,6 +190,8 @@ public void testReplaceServerKeystore() throws IOException, ServiceException, final Path trustStorePath = Paths.get(CONF.get(X509Util.TLS_CONFIG_TRUSTSTORE_LOCATION)); createAndStartFileWatcher(trustStorePath, latch, Duration.ofMillis(20)); + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) + // Replace keystore x509TestContext.regenerateStores(keyType, keyType, storeFileType, storeFileType); @@ -240,6 +242,8 @@ public void testReplaceClientAndServerKeystore() throws GeneralSecurityException final Path trustStorePath = Paths.get(CONF.get(X509Util.TLS_CONFIG_TRUSTSTORE_LOCATION)); createAndStartFileWatcher(trustStorePath, latch, Duration.ofMillis(20)); + Thread.sleep(1100L); // Ensure mtime changes on Java 8 (second granularity) + // Replace keystore and cancel client connections x509TestContext.regenerateStores(keyType, keyType, storeFileType, storeFileType); client.cancelConnections( From af11cf6eab87993f6539635df5ce84f0097af6f6 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 26 Aug 2025 09:49:07 +0100 Subject: [PATCH 027/336] HBASE-29493 Triage TestBucketCacheRefCnt.testInBucketCache intermittent failure caused by RAMCache draining in between (#7205) (#7206) Signed-off-by: Wellington Chevreuil Co-authored-by: Umesh <9414umeshkumar@gmail.com> --- .../hbase/io/hfile/bucket/TestBucketCacheRefCnt.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCacheRefCnt.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCacheRefCnt.java index e71817e6a3f7..4ee3f37819f7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCacheRefCnt.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCacheRefCnt.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.BlockCacheUtil; @@ -229,6 +230,10 @@ public void testInBucketCache() throws IOException { cache.cacheBlock(key, blk); assertTrue(blk.refCnt() == 1 || blk.refCnt() == 2); + // wait for block to move to backing map because refCnt get refreshed once block moves to + // backing map + Waiter.waitFor(HBaseConfiguration.create(), 12000, () -> isRamCacheDrained(key, cache)); + Cacheable block1 = cache.getBlock(key, false, false, false); assertTrue(block1.refCnt() >= 2); assertTrue(((HFileBlock) block1).getByteBuffAllocator() == alloc); @@ -262,6 +267,10 @@ public void testInBucketCache() throws IOException { } } + private boolean isRamCacheDrained(BlockCacheKey key, BucketCache cache) { + return cache.backingMap.containsKey(key) && !cache.ramCache.containsKey(key); + } + @Test public void testMarkStaleAsEvicted() throws Exception { cache = create(1, 1000); From 46631c5106cbef3671453f87bc036d07c98dde89 Mon Sep 17 00:00:00 2001 From: Peng Lu Date: Tue, 26 Aug 2025 22:11:32 +0800 Subject: [PATCH 028/336] HBASE-29184 The snapshot type for disabled table is incorrect when snapshot procedure is enabled (#6790) (#7244) Signed-off-by: Pankaj Kumar Signed-off-by: Chandra Kambham --- .../master/procedure/SnapshotProcedure.java | 5 +- .../procedure/TestSnapshotProcedure.java | 3 + .../TestSnapshotProcedureForSnapshotType.java | 94 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureForSnapshotType.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java index 17a9c083896a..767ae12dfb20 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription.Type; /** * A procedure used to take snapshot on tables. @@ -121,14 +122,16 @@ protected Flow executeFromState(MasterProcedureEnv env, SnapshotState state) setNextState(SnapshotState.SNAPSHOT_WRITE_SNAPSHOT_INFO); return Flow.HAS_MORE_STATE; case SNAPSHOT_WRITE_SNAPSHOT_INFO: - SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, workingDirFS); TableState tableState = env.getMasterServices().getTableStateManager().getTableState(snapshotTable); if (tableState.isEnabled()) { setNextState(SnapshotState.SNAPSHOT_SNAPSHOT_ONLINE_REGIONS); } else if (tableState.isDisabled()) { + // Set the snapshot type to DISABLED as the table is in DISABLED state + snapshot = snapshot.toBuilder().setType(Type.DISABLED).build(); setNextState(SnapshotState.SNAPSHOT_SNAPSHOT_CLOSED_REGIONS); } + SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, workingDirFS); return Flow.HAS_MORE_STATE; case SNAPSHOT_SNAPSHOT_ONLINE_REGIONS: addChildProcedure(createRemoteSnapshotProcedures(env)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java index 8e85490601f2..dec9c5a3edb5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedure.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.client.SnapshotType; import org.apache.hadoop.hbase.client.Table; @@ -72,6 +73,7 @@ public class TestSnapshotProcedure { protected String SNAPSHOT_NAME; protected SnapshotDescription snapshot; protected SnapshotProtos.SnapshotDescription snapshotProto; + protected Admin admin; public static final class DelaySnapshotProcedure extends SnapshotProcedure { public DelaySnapshotProcedure() { @@ -109,6 +111,7 @@ public void setup() throws Exception { config.setInt(RemoteProcedureDispatcher.DISPATCH_MAX_QUEUE_SIZE_CONF_KEY, 128); TEST_UTIL.startMiniCluster(3); master = TEST_UTIL.getHBaseCluster().getMaster(); + admin = TEST_UTIL.getAdmin(); TABLE_NAME = TableName.valueOf(Bytes.toBytes("SPTestTable")); CF = Bytes.toBytes("cf"); SNAPSHOT_NAME = "SnapshotProcedureTest"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureForSnapshotType.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureForSnapshotType.java new file mode 100644 index 000000000000..b876acfb8428 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureForSnapshotType.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Optional; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.SnapshotType; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestSnapshotProcedureForSnapshotType extends TestSnapshotProcedure { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSnapshotProcedureForSnapshotType.class); + + @Rule + public TestName name = new TestName(); + + @Test + public void testSnapshotTypeForEnabledTable() throws IOException { + TableName tableName = TableName.valueOf(name.getMethodName()); + String snapshotName = "snapshot_" + name.getMethodName(); + TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("info")).build(); + tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); + admin.createTable(tableDescriptorBuilder.build()); + + assertTrue(admin.tableExists(tableName)); + assertTrue(admin.isTableEnabled(tableName)); + + admin.snapshot(snapshotName, tableName); + Optional optional = + admin.listSnapshots().stream().filter(s -> snapshotName.equals(s.getName())).findFirst(); + assertTrue(optional.isPresent()); + SnapshotDescription snapshotDescription = optional.get(); + assertEquals(SnapshotType.FLUSH, snapshotDescription.getType()); + } + + @Test + public void testSnapshotTypeForDisabledTable() throws IOException { + TableName tableName = TableName.valueOf(name.getMethodName()); + String snapshotName = "snapshot_" + name.getMethodName(); + TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("info")).build(); + tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); + admin.createTable(tableDescriptorBuilder.build()); + assertTrue(admin.tableExists(tableName)); + assertTrue(admin.isTableEnabled(tableName)); + + admin.disableTable(tableName); + assertTrue(admin.isTableDisabled(tableName)); + + admin.snapshot(snapshotName, tableName); + Optional optional = + admin.listSnapshots().stream().filter(s -> snapshotName.equals(s.getName())).findFirst(); + assertTrue(optional.isPresent()); + SnapshotDescription snapshotDescription = optional.get(); + assertEquals(SnapshotType.DISABLED, snapshotDescription.getType()); + } +} From 213f34179487f003e157ce567605ac877ba2b154 Mon Sep 17 00:00:00 2001 From: Kevin Geiszler Date: Tue, 26 Aug 2025 15:11:22 -0700 Subject: [PATCH 029/336] HBASE-29503: IntegrationTestBackupRestore is passing even if an exception occurs in the thread(s) it creates (#7203) (#7226) (#7247) Signed-off-by: Tak Lon (Stephen) Wu --- .../hbase/IntegrationTestBackupRestore.java | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java index 4326c9852e35..0e18cb491ce2 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java @@ -101,6 +101,38 @@ public class IntegrationTestBackupRestore extends IntegrationTestBase { private static String BACKUP_ROOT_DIR = "backupIT"; + /* + * This class is used to run the backup and restore thread(s). Throwing an exception in this + * thread will not cause the test to fail, so the purpose of this class is to both kick off the + * backup and restore and record any exceptions that occur so they can be thrown in the main + * thread. + */ + protected class BackupAndRestoreThread implements Runnable { + private final TableName table; + private Exception exc; + + public BackupAndRestoreThread(TableName table) { + this.table = table; + this.exc = null; + } + + public Exception getException() { + return this.exc; + } + + @Override + public void run() { + try { + runTestSingle(this.table); + } catch (Exception e) { + LOG.error( + "An exception occurred in thread {} when performing a backup and restore with table {}: ", + Thread.currentThread().getName(), this.table.getNameAsString(), e); + this.exc = e; + } + } + } + @Override @Before public void setUp() throws Exception { @@ -174,28 +206,35 @@ public void testBackupRestore() throws Exception { runTestMulti(); } - private void runTestMulti() throws IOException { + private void runTestMulti() throws Exception { LOG.info("IT backup & restore started"); Thread[] workers = new Thread[numTables]; + BackupAndRestoreThread[] backupAndRestoreThreads = new BackupAndRestoreThread[numTables]; for (int i = 0; i < numTables; i++) { final TableName table = tableNames[i]; - Runnable r = new Runnable() { - @Override - public void run() { - try { - runTestSingle(table); - } catch (IOException e) { - LOG.error("Failed", e); - Assert.fail(e.getMessage()); - } - } - }; - workers[i] = new Thread(r); + BackupAndRestoreThread backupAndRestoreThread = new BackupAndRestoreThread(table); + backupAndRestoreThreads[i] = backupAndRestoreThread; + workers[i] = new Thread(backupAndRestoreThread); workers[i].start(); } - // Wait all workers to finish - for (Thread t : workers) { - Uninterruptibles.joinUninterruptibly(t); + // Wait for all workers to finish and check for errors + Exception error = null; + Exception threadExc; + for (int i = 0; i < numTables; i++) { + Uninterruptibles.joinUninterruptibly(workers[i]); + threadExc = backupAndRestoreThreads[i].getException(); + if (threadExc == null) { + continue; + } + if (error == null) { + error = threadExc; + } else { + error.addSuppressed(threadExc); + } + } + // Throw any found errors after all threads have completed + if (error != null) { + throw error; } LOG.info("IT backup & restore finished"); } @@ -229,8 +268,7 @@ private void loadData(TableName table, int numRows) throws IOException { } private String backup(BackupRequest request, BackupAdmin client) throws IOException { - String backupId = client.backupTables(request); - return backupId; + return client.backupTables(request); } private void restore(RestoreRequest request, BackupAdmin client) throws IOException { @@ -300,7 +338,6 @@ private void runTestSingle(TableName table) throws IOException { private void restoreVerifyTable(Connection conn, BackupAdmin client, TableName table, String backupId, long expectedRows) throws IOException { - TableName[] tablesRestoreIncMultiple = new TableName[] { table }; restore( createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tablesRestoreIncMultiple, null, true), From 198562910aba000245b8e546fadb621e750f111d Mon Sep 17 00:00:00 2001 From: Kevin Geiszler Date: Wed, 27 Aug 2025 09:44:41 -0700 Subject: [PATCH 030/336] HBASE-29507: IntegrationTestBackupRestore is failing because it cannot restore from backup directory (#7240) (#7234) (#7250) Signed-off-by: Tak Lon (Stephen) Wu --- .../org/apache/hadoop/hbase/IntegrationTestBackupRestore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java index 0e18cb491ce2..50131f7c2bf6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java @@ -84,7 +84,7 @@ public class IntegrationTestBackupRestore extends IntegrationTestBase { protected static final int DEFAULT_REGIONSERVER_COUNT = 5; protected static final int DEFAULT_NUMBER_OF_TABLES = 1; protected static final int DEFAULT_NUM_ITERATIONS = 10; - protected static final int DEFAULT_ROWS_IN_ITERATION = 500000; + protected static final int DEFAULT_ROWS_IN_ITERATION = 10000; protected static final String SLEEP_TIME_KEY = "sleeptime"; // short default interval because tests don't run very long. protected static final long SLEEP_TIME_DEFAULT = 50000L; From 2baf739f976cb2a55e7be0bb6b045f12668ba9a6 Mon Sep 17 00:00:00 2001 From: Kevin Geiszler Date: Wed, 27 Aug 2025 11:47:35 -0700 Subject: [PATCH 031/336] HBASE-29544: Assertion errors in BackupAndRestoreThread are not causing IntegrationTestBackupRestore to fail (#7243) (#7252) Signed-off-by: Tak Lon (Stephen) Wu --- .../hbase/IntegrationTestBackupRestore.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java index 50131f7c2bf6..80785b866840 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java @@ -109,26 +109,26 @@ public class IntegrationTestBackupRestore extends IntegrationTestBase { */ protected class BackupAndRestoreThread implements Runnable { private final TableName table; - private Exception exc; + private Throwable throwable; public BackupAndRestoreThread(TableName table) { this.table = table; - this.exc = null; + this.throwable = null; } - public Exception getException() { - return this.exc; + public Throwable getThrowable() { + return this.throwable; } @Override public void run() { try { runTestSingle(this.table); - } catch (Exception e) { + } catch (Throwable t) { LOG.error( - "An exception occurred in thread {} when performing a backup and restore with table {}: ", - Thread.currentThread().getName(), this.table.getNameAsString(), e); - this.exc = e; + "An error occurred in thread {} when performing a backup and restore with table {}: ", + Thread.currentThread().getName(), this.table.getNameAsString(), t); + this.throwable = t; } } } @@ -218,23 +218,23 @@ private void runTestMulti() throws Exception { workers[i].start(); } // Wait for all workers to finish and check for errors - Exception error = null; - Exception threadExc; + Throwable error = null; + Throwable threadThrowable; for (int i = 0; i < numTables; i++) { Uninterruptibles.joinUninterruptibly(workers[i]); - threadExc = backupAndRestoreThreads[i].getException(); - if (threadExc == null) { + threadThrowable = backupAndRestoreThreads[i].getThrowable(); + if (threadThrowable == null) { continue; } if (error == null) { - error = threadExc; + error = threadThrowable; } else { - error.addSuppressed(threadExc); + error.addSuppressed(threadThrowable); } } // Throw any found errors after all threads have completed if (error != null) { - throw error; + throw new AssertionError("An error occurred in a backup and restore thread", error); } LOG.info("IT backup & restore finished"); } From 9dbd6962199abbae19d480ccca2b353cecf14de9 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Thu, 28 Aug 2025 07:26:58 -0400 Subject: [PATCH 032/336] HBASE-29502: Skip meta cache in RegionReplicaReplicationEndpoint when only one replica found (#7202) Signed-off by: Wellington Ramos Chevreuil Signed-off by: Chandra Sekhar K Signed-off by: Nick Dimiduk --- .../RegionReplicaReplicationEndpoint.java | 13 ++- .../TestRegionReplicaReplicationEndpoint.java | 100 ++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java index 754811ce0e04..7dbb4f008f85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java @@ -427,6 +427,7 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, // invalidate the cache and check from meta RegionLocations locations = null; boolean useCache = true; + int retries = 0; while (true) { // get the replicas of the primary region try { @@ -439,12 +440,13 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, // Replicas can take a while to come online. The cache may have only the primary. If we // keep going to the cache, we will not learn of the replicas and their locations after // they come online. - if (useCache && locations.size() == 1 && TableName.isMetaTableName(tableName)) { - if (tableDescriptors.get(tableName).getRegionReplication() > 1) { + if (useCache && locations.size() == 1) { + if (tableDescriptors.get(tableName).getRegionReplication() > 1 && retries <= 3) { // Make an obnoxious log here. See how bad this issue is. Add a timer if happening // too much. LOG.info("Skipping location cache; only one location found for {}", tableName); useCache = false; + retries++; continue; } } @@ -488,6 +490,13 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, } if (locations.size() == 1) { + if (LOG.isTraceEnabled()) { + LOG.trace("Skipping {} entries in table {} because only one region location was found", + entries.size(), tableName); + for (Entry entry : entries) { + LOG.trace("Skipping: {}", entry); + } + } return; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java index 9a03536f7541..ffca0caabef6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java @@ -507,6 +507,106 @@ public void testRegionReplicaReplicationIgnores(boolean dropTable, boolean disab } } + @Test + public void testMetaCacheMissTriggersRefresh() throws Exception { + TableName tableName = TableName.valueOf(name.getMethodName()); + int regionReplication = 3; + HTableDescriptor htd = HTU.createTableDescriptor(tableName); + htd.setRegionReplication(regionReplication); + createOrEnableTableWithRetries(htd, true); + + Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); + Table table = connection.getTable(tableName); + + try { + HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 100); + + RegionLocator rl = connection.getRegionLocator(tableName); + HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); + rl.close(); + + AtomicLong skippedEdits = new AtomicLong(); + RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink = + mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class); + when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits); + + FSTableDescriptors fstd = + new FSTableDescriptors(FileSystem.get(HTU.getConfiguration()), HTU.getDefaultRootDirPath()); + + RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter = + new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink, + (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, + fstd); + + Cell cell = CellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("testRow")).setFamily(HBaseTestingUtility.fam1) + .setValue(Bytes.toBytes("testValue")).setType(Type.Put).build(); + + Entry entry = + new Entry(new WALKeyImpl(encodedRegionName, tableName, 1), new WALEdit().add(cell)); + + sinkWriter.append(tableName, encodedRegionName, Bytes.toBytes("testRow"), + Lists.newArrayList(entry)); + + assertEquals("No entries should be skipped for valid table", 0, skippedEdits.get()); + + } finally { + table.close(); + connection.close(); + } + } + + @Test + public void testMetaCacheSkippedForSingleReplicaTable() throws Exception { + TableName tableName = TableName.valueOf(name.getMethodName()); + int regionReplication = 1; + HTableDescriptor htd = HTU.createTableDescriptor(tableName); + htd.setRegionReplication(regionReplication); + createOrEnableTableWithRetries(htd, true); + + Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); + Table table = connection.getTable(tableName); + + try { + HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 100); + + RegionLocator rl = connection.getRegionLocator(tableName); + HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); + rl.close(); + + AtomicLong skippedEdits = new AtomicLong(); + RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink = + mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class); + when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits); + + FSTableDescriptors fstd = + new FSTableDescriptors(FileSystem.get(HTU.getConfiguration()), HTU.getDefaultRootDirPath()); + + RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter = + new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink, + (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, + fstd); + + Cell cell = CellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("testRow")).setFamily(HBaseTestingUtility.fam1) + .setValue(Bytes.toBytes("testValue")).setType(Type.Put).build(); + + Entry entry = + new Entry(new WALKeyImpl(encodedRegionName, tableName, 1), new WALEdit().add(cell)); + + sinkWriter.append(tableName, encodedRegionName, Bytes.toBytes("testRow"), + Lists.newArrayList(entry)); + + assertEquals("No entries should be skipped for single replica table", 0, skippedEdits.get()); + + } finally { + table.close(); + connection.close(); + } + } + private void createOrEnableTableWithRetries(TableDescriptor htd, boolean createTableOperation) { // Helper function to run create/enable table operations with a retry feature boolean continueToRetry = true; From 4d77872cbb65c256a43f99d8522605da41e0c3a8 Mon Sep 17 00:00:00 2001 From: alexdongli0829 <40448063+alexdongli0829@users.noreply.github.com> Date: Fri, 29 Aug 2025 11:43:09 +1000 Subject: [PATCH 033/336] HBASE-29532 Fix the potential NPE issue when the specific recover edit path set (#7231) Co-authored-by: Dong Li Signed-off-by: Duo Zhang (cherry picked from commit 3b4c023e0b4dabdf1c83c7a1e9e60c346062d180) --- .../java/org/apache/hadoop/hbase/regionserver/HRegion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 164675fe2a1c..e550e8ba8df5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5576,7 +5576,7 @@ private long replayRecoveredEdits(final Path edits, Map maxSeqIdIn } } catch (EOFException eof) { if (!conf.getBoolean(RECOVERED_EDITS_IGNORE_EOF, false)) { - Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits); + Path p = WALSplitUtil.moveAsideBadEditsFile(fs, edits); msg = "EnLongAddered EOF. Most likely due to Master failure during " + "wal splitting, so we have this data in another edit. Continuing, but renaming " + edits + " as " + p + " for region " + this; @@ -5590,7 +5590,7 @@ private long replayRecoveredEdits(final Path edits, Map maxSeqIdIn // If the IOE resulted from bad file format, // then this problem is idempotent and retrying won't help if (ioe.getCause() instanceof ParseException) { - Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits); + Path p = WALSplitUtil.moveAsideBadEditsFile(fs, edits); msg = "File corruption enLongAddered! " + "Continuing, but renaming " + edits + " as " + p; LOG.warn(msg, ioe); From fa00999f429ed5145d92a82c5c8759c6c2a42d0d Mon Sep 17 00:00:00 2001 From: Sreenivasulu Date: Sun, 31 Aug 2025 18:43:35 +0530 Subject: [PATCH 034/336] HBASE-29431 Update the 'ExcludeDNs' information with the cause in RS UI (#7128) Signed-off-by: Duo Zhang Signed-off-by: Pankaj Kumar Signed-off-by: Chandra Kambham (cherry picked from commit d07ed70ac3cc4d6cd37cec12b88cb32955dbfea6) --- .../FanOutOneBlockAsyncDFSOutputHelper.java | 3 +- .../monitor/ExcludeDatanodeManager.java | 31 +++++++++++++++++-- .../io/asyncfs/monitor/StreamSlowMonitor.java | 3 +- .../MetricsRegionServerWrapperImpl.java | 5 +-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java index b93768ae0849..81716182f689 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java @@ -647,7 +647,8 @@ private static FanOutOneBlockAsyncDFSOutput createOutput(DistributedFileSystem d } catch (Exception e) { // exclude the broken DN next time toExcludeNodes.add(datanodeInfo); - excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "connect error"); + excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, + ExcludeDatanodeManager.ExcludeCause.CONNECT_ERROR.getCause()); throw e; } } diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java index 61f75582a1c9..7bed67a94be5 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/ExcludeDatanodeManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -52,7 +53,7 @@ public class ExcludeDatanodeManager implements ConfigurationObserver { "hbase.regionserver.async.wal.exclude.datanode.info.ttl.hour"; public static final int DEFAULT_WAL_EXCLUDE_DATANODE_TTL = 6; // 6 hours - private volatile Cache excludeDNsCache; + private volatile Cache> excludeDNsCache; private final int maxExcludeDNCount; private final Configuration conf; // This is a map of providerId->StreamSlowMonitor @@ -78,7 +79,7 @@ public ExcludeDatanodeManager(Configuration conf) { public boolean tryAddExcludeDN(DatanodeInfo datanodeInfo, String cause) { boolean alreadyMarkedSlow = getExcludeDNs().containsKey(datanodeInfo); if (!alreadyMarkedSlow) { - excludeDNsCache.put(datanodeInfo, EnvironmentEdgeManager.currentTime()); + excludeDNsCache.put(datanodeInfo, new Pair<>(cause, EnvironmentEdgeManager.currentTime())); LOG.info( "Added datanode: {} to exclude cache by [{}] success, current excludeDNsCache size={}", datanodeInfo, cause, excludeDNsCache.size()); @@ -95,7 +96,31 @@ public StreamSlowMonitor getStreamSlowMonitor(String name) { return streamSlowMonitors.computeIfAbsent(key, k -> new StreamSlowMonitor(conf, key, this)); } - public Map getExcludeDNs() { + /** + * Enumerates the reasons for excluding a datanode from certain operations. Each enum constant + * represents a specific cause leading to exclusion. + */ + public enum ExcludeCause { + CONNECT_ERROR("connect error"), + SLOW_PACKET_ACK("slow packet ack"); + + private final String cause; + + ExcludeCause(String cause) { + this.cause = cause; + } + + public String getCause() { + return cause; + } + + @Override + public String toString() { + return cause; + } + } + + public Map> getExcludeDNs() { return excludeDNsCache.asMap(); } diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java index c415706aa6af..a4b80fc64560 100644 --- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java +++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java @@ -156,7 +156,8 @@ public void checkProcessTimeAndSpeed(DatanodeInfo datanodeInfo, long packetDataL + "lastAckTimestamp={}, monitor name: {}", datanodeInfo, packetDataLen, processTimeMs, unfinished, lastAckTimestamp, this.name); if (addSlowAckData(datanodeInfo, packetDataLen, processTimeMs)) { - excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "slow packet ack"); + excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, + ExcludeDatanodeManager.ExcludeCause.SLOW_PACKET_ACK.getCause()); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 2bd396242a17..d52509e08088 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -433,8 +433,9 @@ public List getWALExcludeDNs() { if (excludeDatanodeManager == null) { return Collections.emptyList(); } - return excludeDatanodeManager.getExcludeDNs().entrySet().stream() - .map(e -> e.getKey().toString() + ", " + e.getValue()).collect(Collectors.toList()); + return excludeDatanodeManager.getExcludeDNs().entrySet().stream().map(e -> e.getKey().toString() + + " - " + e.getValue().getSecond() + " - " + e.getValue().getFirst()) + .collect(Collectors.toList()); } @Override From eee5dd3c2e50d925104f817db1a5f16311314635 Mon Sep 17 00:00:00 2001 From: Ariadne-team <3631176787@qq.com> Date: Sun, 31 Aug 2025 22:23:55 +0800 Subject: [PATCH 035/336] HBASE-28866 Setting `hbase.oldwals.cleaner.thread.size` to negative value will break HMaster and produce hard-to-diagnose logs (#6310) Co-authored-by: AlphaDora <598669236@qq.com> Signed-off-by: Duo Zhang (cherry picked from commit 8a7defb858a618dbc18ebe528b4b12fa17cc6f72) --- .../org/apache/hadoop/hbase/master/cleaner/LogCleaner.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java index ac0a98801c15..6ede2b50d8a3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java @@ -78,6 +78,13 @@ public LogCleaner(final int period, final Stoppable stopper, Configuration conf, pool, params, null); this.pendingDelete = new LinkedBlockingQueue<>(); int size = conf.getInt(OLD_WALS_CLEANER_THREAD_SIZE, DEFAULT_OLD_WALS_CLEANER_THREAD_SIZE); + if (size <= 0) { + size = DEFAULT_OLD_WALS_CLEANER_THREAD_SIZE; + LOG.warn( + "The configuration {} has been set to an invalid value {}, " + + "the default value {} will be used.", + OLD_WALS_CLEANER_THREAD_SIZE, size, DEFAULT_OLD_WALS_CLEANER_THREAD_SIZE); + } this.oldWALsCleaner = createOldWalsCleaner(size); this.cleanerThreadTimeoutMsec = conf.getLong(OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC, DEFAULT_OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC); From bf355e6a1969140859a62da76a12c68b0ab1766e Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Tue, 2 Sep 2025 07:21:22 +0200 Subject: [PATCH 036/336] HBASE-29549 Mockito failures in TestServerCall with Java 21 (#7254) Signed-off-by: Nihal Jain (cherry picked from commit e65dfa8adf23b71827ce17c73b3724d9557fc569) --- .../org/apache/hadoop/hbase/ipc/TestServerCall.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java index 46239e95859b..152f358b1a63 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestServerCall.java @@ -67,7 +67,7 @@ public class TestServerCall { private Message mockParam; private ByteBuffAllocator mockAllocator; private CellBlockBuilder mockCellBlockBuilder; - private InetAddress mockAddr; + private InetAddress lbAddr; private BlockingService mockService; private MethodDescriptor mockMethodDescriptor; @@ -80,7 +80,7 @@ public void setUp() throws Exception { mockParam = mock(Message.class); mockAllocator = mock(ByteBuffAllocator.class); mockCellBlockBuilder = mock(CellBlockBuilder.class); - mockAddr = mock(InetAddress.class); + lbAddr = InetAddress.getLoopbackAddress(); mockMethodDescriptor = org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.getDescriptor() @@ -105,7 +105,7 @@ public void testSetResponseWithIOException() throws Exception { // Create NettyServerCall instance NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, - mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockParam, null, mockConnection, 100, lbAddr, System.currentTimeMillis(), 60000, mockAllocator, failingCellBlockBuilder, null); // Set a successful response, but CellBlockBuilder will fail @@ -138,7 +138,7 @@ public void testSetResponseWithDoubleIOException() throws Exception { any(), any()); NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, - mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockParam, null, mockConnection, 100, lbAddr, System.currentTimeMillis(), 60000, mockAllocator, failingCellBlockBuilder, null); Message mockResponse = mock(Message.class); @@ -158,7 +158,7 @@ public void testSetResponseNormalFlow() throws Exception { when(normalCellBlockBuilder.buildCellBlock(any(), any(), any())).thenReturn(null); NettyServerCall call = new NettyServerCall(1, mockService, mockMethodDescriptor, header, - mockParam, null, mockConnection, 100, mockAddr, System.currentTimeMillis(), 60000, + mockParam, null, mockConnection, 100, lbAddr, System.currentTimeMillis(), 60000, mockAllocator, normalCellBlockBuilder, null); RPCProtos.CellBlockMeta mockResponse = From bf25fde27d73a6e92d9a7fe1a731238e20598d61 Mon Sep 17 00:00:00 2001 From: Ariadne-team <3631176787@qq.com> Date: Wed, 3 Sep 2025 09:44:29 +0800 Subject: [PATCH 037/336] HBASE-28881 Adding check and diagnose log for "hbase.master.procedure.threads" when set to non-positive value (#7267) Signed-off-by: Duo Zhang (cherry picked from commit 6d739b77a66be3643128540e1ce711b74d65a5b7) --- .../java/org/apache/hadoop/hbase/master/HMaster.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index e099760a7a84..35f212b44597 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1803,8 +1803,15 @@ public void abortProcess() { configurationManager.registerObserver(procEnv); int cpus = Runtime.getRuntime().availableProcessors(); - final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max( - (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); + int defaultNumThreads = Math.max((cpus > 0 ? cpus / 4 : 0), + MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS); + int numThreads = + conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, defaultNumThreads); + if (numThreads <= 0) { + LOG.warn("{} is set to {}, which is invalid, using default value {} instead", + MasterProcedureConstants.MASTER_PROCEDURE_THREADS, numThreads, defaultNumThreads); + numThreads = defaultNumThreads; + } final boolean abortOnCorruption = conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); From 9698c236ca1be853f4ae7ff022ce4942ed02e227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Paksy?= Date: Wed, 3 Sep 2025 11:28:30 +0200 Subject: [PATCH 038/336] HBASE-29556: Display HBCK and CatalogJanitor report errors properly on HBCK Report page (#7255) Signed-off-by: Nihal Jain (cherry picked from commit 394db1686724336c3afa243b1f0a70f8c5fe4ab8) --- .../resources/hbase-webapps/master/hbck.jsp | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp index 38e16ca8e28f..a6c6c2d17e66 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp @@ -39,21 +39,24 @@ <%@ page import="org.apache.hadoop.hbase.master.janitor.CatalogJanitorReport" %> <%@ page import="java.util.Optional" %> <%@ page import="org.apache.hadoop.hbase.util.EnvironmentEdgeManager" %> +<%@ page import="org.apache.hbase.thirdparty.com.google.protobuf.ServiceException" %> <% final String cacheParameterValue = request.getParameter("cache"); final HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); pageContext.setAttribute("pageTitle", "HBase Master HBCK Report: " + master.getServerName()); + String hbckChoreErrorMessage = null; + String catalogJanitorErrorMessage = null; if (!Boolean.parseBoolean(cacheParameterValue)) { // Run the two reporters inline w/ drawing of the page. If exception, will show in page draw. try { master.getMasterRpcServices().runHbckChore(null, null); - } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) { - out.write("Failed generating a new hbck_chore report; using cache; try again or run hbck_chore_run in the shell: " + se.getMessage() + "\n"); + } catch (ServiceException se) { + hbckChoreErrorMessage = "Failed generating a new hbck_chore report; using cache; try again or run hbck_chore_run in the shell: " + se.getMessage(); } try { master.getMasterRpcServices().runCatalogScan(null, null); - } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException se) { - out.write("Failed generating a new catalogjanitor report; using cache; try again or run catalogjanitor_run in the shell: " + se.getMessage() + "\n"); + } catch (ServiceException se) { + catalogJanitorErrorMessage = "Failed generating a new catalogjanitor report; using cache; try again or run catalogjanitor_run in the shell: " + se.getMessage(); } } HbckChore hbckChore = master.getHbckChore(); @@ -119,6 +122,12 @@ + <% if(hbckChoreErrorMessage != null) { %> +

+ <% } %> + <% if (hbckReport != null && hbckReport.getInconsistentRegions().size() > 0) { %>
@@ -230,6 +239,13 @@ <% } %>
+ + <% if(catalogJanitorErrorMessage != null) { %> + + <% } %> + <% if (cjReport != null && !cjReport.isEmpty()) { %> <% if (!cjReport.getHoles().isEmpty()) { %>
From c72760abaf32b223ac4032bd1dff74590a1c64e6 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Wed, 3 Sep 2025 13:52:45 -0400 Subject: [PATCH 039/336] HBASE-29479: QuotaCache should always return accurate information (#7188) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/QuotaCache.java | 209 ++++++++++-------- .../hbase/quotas/TestDefaultAtomicQuota.java | 9 - .../hadoop/hbase/quotas/TestQuotaCache.java | 118 +++++++++- 3 files changed, 227 insertions(+), 109 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index d77f6219ae59..2ec9d049f7da 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent; - import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -30,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; @@ -56,10 +55,7 @@ /** * Cache that keeps track of the quota settings for the users and tables that are interacting with - * it. To avoid blocking the operations if the requested quota is not in cache an "empty quota" will - * be returned and the request to fetch the quota information will be enqueued for the next refresh. - * TODO: At the moment the Cache has a Chore that will be triggered every 5min or on cache-miss - * events. Later the Quotas will be pushed using the notification system. + * it. */ @InterfaceAudience.Private @InterfaceStability.Evolving @@ -100,6 +96,62 @@ public class QuotaCache implements Stoppable { private QuotaRefresherChore refreshChore; private boolean stopped = true; + private final Fetcher userQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String user) { + final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); + final Set tables = QuotaCache.this.tableQuotaCache.keySet(); + return QuotaUtil.makeGetForUserQuotas(user, tables, namespaces); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, tableMachineQuotaFactors, + machineQuotaFactor); + } + }; + + private final Fetcher regionServerQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String regionServer) { + return QuotaUtil.makeGetForRegionServerQuotas(regionServer); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); + } + }; + + private final Fetcher tableQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final TableName table) { + return QuotaUtil.makeGetForTableQuotas(table); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, + tableMachineQuotaFactors); + } + }; + + private final Fetcher namespaceQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String namespace) { + return QuotaUtil.makeGetForNamespaceQuotas(namespace); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, machineQuotaFactor); + } + }; + public QuotaCache(final RegionServerServices rsServices) { this.rsServices = rsServices; this.userOverrideRequestAttributeKey = @@ -153,8 +205,13 @@ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableNa * @return the quota info associated to specified user */ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { - return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi), - () -> QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); + String user = getQuotaUserName(ugi); + if (!userQuotaCache.containsKey(user)) { + userQuotaCache.put(user, + QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); + fetch("user", userQuotaCache, userQuotaStateFetcher); + } + return userQuotaCache.get(user); } /** @@ -163,7 +220,11 @@ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { * @return the limiter associated to the specified table */ public QuotaLimiter getTableLimiter(final TableName table) { - return getQuotaState(this.tableQuotaCache, table).getGlobalLimiter(); + if (!tableQuotaCache.containsKey(table)) { + tableQuotaCache.put(table, new QuotaState()); + fetch("table", tableQuotaCache, tableQuotaStateFetcher); + } + return tableQuotaCache.get(table).getGlobalLimiter(); } /** @@ -172,7 +233,11 @@ public QuotaLimiter getTableLimiter(final TableName table) { * @return the limiter associated to the specified namespace */ public QuotaLimiter getNamespaceLimiter(final String namespace) { - return getQuotaState(this.namespaceQuotaCache, namespace).getGlobalLimiter(); + if (!namespaceQuotaCache.containsKey(namespace)) { + namespaceQuotaCache.put(namespace, new QuotaState()); + fetch("namespace", namespaceQuotaCache, namespaceQuotaStateFetcher); + } + return namespaceQuotaCache.get(namespace).getGlobalLimiter(); } /** @@ -181,13 +246,41 @@ public QuotaLimiter getNamespaceLimiter(final String namespace) { * @return the limiter associated to the specified region server */ public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) { - return getQuotaState(this.regionServerQuotaCache, regionServer).getGlobalLimiter(); + if (!regionServerQuotaCache.containsKey(regionServer)) { + regionServerQuotaCache.put(regionServer, new QuotaState()); + fetch("regionServer", regionServerQuotaCache, regionServerQuotaStateFetcher); + } + return regionServerQuotaCache.get(regionServer).getGlobalLimiter(); } protected boolean isExceedThrottleQuotaEnabled() { return exceedThrottleQuotaEnabled; } + private void fetch(final String type, final Map quotasMap, + final Fetcher fetcher) { + // Find the quota entries to update + List gets = quotasMap.keySet().stream().map(fetcher::makeGet).collect(Collectors.toList()); + + // fetch and update the quota entries + if (!gets.isEmpty()) { + try { + for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { + V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); + if (quotaInfo != null) { + quotaInfo.update(entry.getValue()); + } + + if (LOG.isTraceEnabled()) { + LOG.trace("Loading {} key={} quotas={}", type, entry.getKey(), quotaInfo); + } + } + } catch (IOException e) { + LOG.warn("Unable to read {} from quota table", type, e); + } + } + } + /** * Applies a request attribute user override if available, otherwise returns the UGI's short * username @@ -210,14 +303,6 @@ private String getQuotaUserName(final UserGroupInformation ugi) { return Bytes.toString(override); } - /** - * Returns the QuotaState requested. If the quota info is not in cache an empty one will be - * returned and the quota request will be enqueued for the next cache refresh. - */ - private QuotaState getQuotaState(final ConcurrentMap quotasMap, final K key) { - return computeIfAbsent(quotasMap, key, QuotaState::new); - } - void triggerCacheRefresh() { refreshChore.triggerNow(); } @@ -226,10 +311,6 @@ void forceSynchronousCacheRefresh() { refreshChore.chore(); } - long getLastUpdate() { - return refreshChore.lastUpdate; - } - Map getNamespaceQuotaCache() { return namespaceQuotaCache; } @@ -248,8 +329,6 @@ Map getUserQuotaCache() { // TODO: Remove this once we have the notification bus private class QuotaRefresherChore extends ScheduledChore { - private long lastUpdate = 0; - // Querying cluster metrics so often, per-RegionServer, limits horizontal scalability. // So we cache the results to reduce that load. private final RefreshableExpiringValueCache tableRegionStatesClusterMetrics; @@ -307,74 +386,12 @@ protected void chore() { .computeIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, key -> new QuotaState()); updateQuotaFactors(); - fetchNamespaceQuotaState(); - fetchTableQuotaState(); - fetchUserQuotaState(); - fetchRegionServerQuotaState(); + fetchAndEvict("namespace", QuotaCache.this.namespaceQuotaCache, namespaceQuotaStateFetcher); + fetchAndEvict("table", QuotaCache.this.tableQuotaCache, tableQuotaStateFetcher); + fetchAndEvict("user", QuotaCache.this.userQuotaCache, userQuotaStateFetcher); + fetchAndEvict("regionServer", QuotaCache.this.regionServerQuotaCache, + regionServerQuotaStateFetcher); fetchExceedThrottleQuota(); - lastUpdate = EnvironmentEdgeManager.currentTime(); - } - - private void fetchNamespaceQuotaState() { - fetch("namespace", QuotaCache.this.namespaceQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForNamespaceQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, - machineQuotaFactor); - } - }); - } - - private void fetchTableQuotaState() { - fetch("table", QuotaCache.this.tableQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForTableQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors); - } - }); - } - - private void fetchUserQuotaState() { - final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); - final Set tables = QuotaCache.this.tableQuotaCache.keySet(); - fetch("user", QuotaCache.this.userQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForUserQuotas(entry.getKey(), tables, namespaces); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors, machineQuotaFactor); - } - }); - } - - private void fetchRegionServerQuotaState() { - fetch("regionServer", QuotaCache.this.regionServerQuotaCache, - new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForRegionServerQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); - } - }); } private void fetchExceedThrottleQuota() { @@ -386,7 +403,7 @@ private void fetchExceedThrottleQuota() { } } - private void fetch(final String type, + private void fetchAndEvict(final String type, final ConcurrentMap quotasMap, final Fetcher fetcher) { long now = EnvironmentEdgeManager.currentTime(); long evictPeriod = getPeriod() * EVICT_PERIOD_FACTOR; @@ -398,7 +415,7 @@ private void fetch(final String type, if (lastQuery > 0 && (now - lastQuery) >= evictPeriod) { toRemove.add(entry.getKey()); } else { - gets.add(fetcher.makeGet(entry)); + gets.add(fetcher.makeGet(entry.getKey())); } } @@ -543,8 +560,8 @@ static interface ThrowingSupplier { T get() throws Exception; } - static interface Fetcher { - Get makeGet(Map.Entry entry); + interface Fetcher { + Get makeGet(Key key); Map fetchEntries(List gets) throws IOException; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java index 966bce6bcdb9..31840cb8d2f2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java @@ -81,10 +81,6 @@ public static void setUpBeforeClass() throws Exception { @Test public void testDefaultAtomicReadLimits() throws Exception { - // No write throttling - configureLenientThrottle(ThrottleType.ATOMIC_WRITE_SIZE); - refreshQuotas(); - // Should have a strict throttle by default TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100); @@ -102,11 +98,6 @@ public void testDefaultAtomicReadLimits() throws Exception { @Test public void testDefaultAtomicWriteLimits() throws Exception { - // No read throttling - configureLenientThrottle(ThrottleType.ATOMIC_REQUEST_NUMBER); - configureLenientThrottle(ThrottleType.ATOMIC_READ_SIZE); - refreshQuotas(); - // Should have a strict throttle by default TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java index 09e152369121..f4f876f104ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java @@ -20,13 +20,16 @@ import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota; import static org.junit.Assert.assertEquals; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -42,8 +45,8 @@ public class TestQuotaCache { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final int REFRESH_TIME_MS = 1000; - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL); EnvironmentEdgeManager.reset(); TEST_UTIL.shutdownMiniCluster(); @@ -68,7 +71,6 @@ public void testDefaultUserRefreshFrequency() throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); - assertEquals(userQuotaState.getLastUpdate(), 0); QuotaCache.TEST_BLOCK_REFRESH = false; // new user should have refreshed immediately @@ -86,4 +88,112 @@ public void testDefaultUserRefreshFrequency() throws Exception { // should refresh after time has passed TEST_UTIL.waitFor(5_000, () -> lastUpdate != userQuotaState.getLastUpdate()); } + + @Test + public void testUserQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleUser("my_user", ThrottleType.READ_NUMBER, 3737, + TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("my_user"); + QuotaLimiter quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific user quota, fall back to default + ugi = UserGroupInformation.createRemoteUser("my_user2"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(1000, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + ugi = UserGroupInformation.createRemoteUser("my_user"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + ugi = UserGroupInformation.createRemoteUser("my_user2"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(1000, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testTableQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleTable(TableName.valueOf("my_table"), + ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific table quota, fall back to default + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2")); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2")); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testNamespaceQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleNamespace("my_namespace", ThrottleType.READ_NUMBER, + 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific namespace quota, fall back to default + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testRegionServerQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleRegionServer("my_region_server", + ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific server quota, fall back to default + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } } From f667d2172978c95765276afcd2d28ef388aac168 Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Mon, 8 Apr 2024 20:54:19 +0530 Subject: [PATCH 040/336] HBASE-28465 Implementation of framework for time-based priority bucket-cache (#5793) Signed-off-by: Wellington Chevreuil Change-Id: If213337be959a392a9bc55aba63b4d033df8e729 --- .../regionserver/DataTieringException.java | 27 ++ .../regionserver/DataTieringManager.java | 222 ++++++++++ .../hbase/regionserver/DataTieringType.java | 26 ++ .../hbase/regionserver/HRegionServer.java | 1 + .../regionserver/TestDataTieringManager.java | 389 ++++++++++++++++++ 5 files changed, 665 insertions(+) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java new file mode 100644 index 000000000000..8d356422f6e0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class DataTieringException extends Exception { + DataTieringException(String reason) { + super(reason); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java new file mode 100644 index 000000000000..0bc04ddc428b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.util.HashSet; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The DataTieringManager class categorizes data into hot data and cold data based on the specified + * {@link DataTieringType} when DataTiering is enabled. DataTiering is disabled by default with + * {@link DataTieringType} set to {@link DataTieringType#NONE}. The {@link DataTieringType} + * determines the logic for distinguishing data into hot or cold. By default, all data is considered + * as hot. + */ +@InterfaceAudience.Private +public class DataTieringManager { + private static final Logger LOG = LoggerFactory.getLogger(DataTieringManager.class); + public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type"; + public static final String DATATIERING_HOT_DATA_AGE_KEY = + "hbase.hstore.datatiering.hot.age.millis"; + public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE; + public static final long DEFAULT_DATATIERING_HOT_DATA_AGE = 7 * 24 * 60 * 60 * 1000; // 7 Days + private static DataTieringManager instance; + private final Map onlineRegions; + + private DataTieringManager(Map onlineRegions) { + this.onlineRegions = onlineRegions; + } + + /** + * Initializes the DataTieringManager instance with the provided map of online regions. + * @param onlineRegions A map containing online regions. + */ + public static synchronized void instantiate(Map onlineRegions) { + if (instance == null) { + instance = new DataTieringManager(onlineRegions); + LOG.info("DataTieringManager instantiated successfully."); + } else { + LOG.warn("DataTieringManager is already instantiated."); + } + } + + /** + * Retrieves the instance of DataTieringManager. + * @return The instance of DataTieringManager. + * @throws IllegalStateException if DataTieringManager has not been instantiated. + */ + public static synchronized DataTieringManager getInstance() { + if (instance == null) { + throw new IllegalStateException( + "DataTieringManager has not been instantiated. Call instantiate() first."); + } + return instance; + } + + /** + * Determines whether data tiering is enabled for the given block cache key. + * @param key the block cache key + * @return {@code true} if data tiering is enabled for the HFile associated with the key, + * {@code false} otherwise + * @throws DataTieringException if there is an error retrieving the HFile path or configuration + */ + public boolean isDataTieringEnabled(BlockCacheKey key) throws DataTieringException { + Path hFilePath = key.getFilePath(); + if (hFilePath == null) { + throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path"); + } + return isDataTieringEnabled(hFilePath); + } + + /** + * Determines whether data tiering is enabled for the given HFile path. + * @param hFilePath the path to the HFile + * @return {@code true} if data tiering is enabled, {@code false} otherwise + * @throws DataTieringException if there is an error retrieving the configuration + */ + public boolean isDataTieringEnabled(Path hFilePath) throws DataTieringException { + Configuration configuration = getConfiguration(hFilePath); + DataTieringType dataTieringType = getDataTieringType(configuration); + return !dataTieringType.equals(DataTieringType.NONE); + } + + /** + * Determines whether the data associated with the given block cache key is considered hot. + * @param key the block cache key + * @return {@code true} if the data is hot, {@code false} otherwise + * @throws DataTieringException if there is an error retrieving data tiering information or the + * HFile maximum timestamp + */ + public boolean isHotData(BlockCacheKey key) throws DataTieringException { + Path hFilePath = key.getFilePath(); + if (hFilePath == null) { + throw new DataTieringException("BlockCacheKey Doesn't Contain HFile Path"); + } + return isHotData(hFilePath); + } + + /** + * Determines whether the data in the HFile at the given path is considered hot based on the + * configured data tiering type and hot data age. + * @param hFilePath the path to the HFile + * @return {@code true} if the data is hot, {@code false} otherwise + * @throws DataTieringException if there is an error retrieving data tiering information or the + * HFile maximum timestamp + */ + public boolean isHotData(Path hFilePath) throws DataTieringException { + Configuration configuration = getConfiguration(hFilePath); + DataTieringType dataTieringType = getDataTieringType(configuration); + + if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { + long hotDataAge = getDataTieringHotDataAge(configuration); + + HStoreFile hStoreFile = getHStoreFile(hFilePath); + if (hStoreFile == null) { + LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist"); + return false; + } + OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); + if (!maxTimestamp.isPresent()) { + throw new DataTieringException("Maximum timestamp not present for " + hFilePath); + } + + long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime(); + long diff = currentTimestamp - maxTimestamp.getAsLong(); + return diff <= hotDataAge; + } + // DataTieringType.NONE or other types are considered hot by default + return true; + } + + /** + * Returns a set of cold data filenames from the given set of cached blocks. Cold data is + * determined by the configured data tiering type and hot data age. + * @param allCachedBlocks a set of all cached block cache keys + * @return a set of cold data filenames + * @throws DataTieringException if there is an error determining whether a block is hot + */ + public Set getColdDataFiles(Set allCachedBlocks) + throws DataTieringException { + Set coldHFiles = new HashSet<>(); + for (BlockCacheKey key : allCachedBlocks) { + if (coldHFiles.contains(key.getHfileName())) { + continue; + } + if (!isHotData(key)) { + coldHFiles.add(key.getHfileName()); + } + } + return coldHFiles; + } + + private HRegion getHRegion(Path hFilePath) throws DataTieringException { + if (hFilePath.getParent() == null || hFilePath.getParent().getParent() == null) { + throw new DataTieringException("Incorrect HFile Path: " + hFilePath); + } + String regionId = hFilePath.getParent().getParent().getName(); + HRegion hRegion = this.onlineRegions.get(regionId); + if (hRegion == null) { + throw new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist"); + } + return hRegion; + } + + private HStore getHStore(Path hFilePath) throws DataTieringException { + HRegion hRegion = getHRegion(hFilePath); + String columnFamily = hFilePath.getParent().getName(); + HStore hStore = hRegion.getStore(Bytes.toBytes(columnFamily)); + if (hStore == null) { + throw new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist"); + } + return hStore; + } + + private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException { + HStore hStore = getHStore(hFilePath); + for (HStoreFile file : hStore.getStorefiles()) { + if (file.getPath().equals(hFilePath)) { + return file; + } + } + return null; + } + + private Configuration getConfiguration(Path hFilePath) throws DataTieringException { + HStore hStore = getHStore(hFilePath); + return hStore.getReadOnlyConfiguration(); + } + + private DataTieringType getDataTieringType(Configuration conf) { + return DataTieringType.valueOf(conf.get(DATATIERING_KEY, DEFAULT_DATATIERING.name())); + } + + private long getDataTieringHotDataAge(Configuration conf) { + return Long.parseLong( + conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE))); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java new file mode 100644 index 000000000000..ee54576a6487 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Public +public enum DataTieringType { + NONE, + TIME_RANGE +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 621cba3775a0..f1615e5e1e91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -700,6 +700,7 @@ public HRegionServer(final Configuration conf) throws IOException { // no need to instantiate block cache and mob file cache when master not carry table if (!isMasterNotCarryTable) { blockCache = BlockCacheFactory.createBlockCache(conf); + DataTieringManager.instantiate(onlineRegions); mobFileCache = new MobFileCache(conf); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java new file mode 100644 index 000000000000..afb5862a8a46 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * This class is used to test the functionality of the DataTieringManager. + * + * The mock online regions are stored in {@link TestDataTieringManager#testOnlineRegions}. + * For all tests, the setup of {@link TestDataTieringManager#testOnlineRegions} occurs only once. + * Please refer to {@link TestDataTieringManager#setupOnlineRegions()} for the structure. + * Additionally, a list of all store files is maintained in {@link TestDataTieringManager#hStoreFiles}. + * The characteristics of these store files are listed below: + * @formatter:off ## HStoreFile Information + * + * | HStoreFile | Region | Store | DataTiering | isHot | + * |------------------|--------------------|---------------------|-----------------------|-------| + * | hStoreFile0 | region1 | hStore11 | TIME_RANGE | true | + * | hStoreFile1 | region1 | hStore12 | NONE | true | + * | hStoreFile2 | region2 | hStore21 | TIME_RANGE | true | + * | hStoreFile3 | region2 | hStore22 | TIME_RANGE | false | + * @formatter:on + */ + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestDataTieringManager { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDataTieringManager.class); + + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static Configuration defaultConf; + private static FileSystem fs; + private static CacheConfig cacheConf; + private static Path testDir; + private static Map testOnlineRegions; + + private static DataTieringManager dataTieringManager; + private static List hStoreFiles; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName()); + defaultConf = TEST_UTIL.getConfiguration(); + fs = HFileSystem.get(defaultConf); + BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf); + cacheConf = new CacheConfig(defaultConf, blockCache); + setupOnlineRegions(); + DataTieringManager.instantiate(testOnlineRegions); + dataTieringManager = DataTieringManager.getInstance(); + } + + @FunctionalInterface + interface DataTieringMethodCallerWithPath { + boolean call(DataTieringManager manager, Path path) throws DataTieringException; + } + + @FunctionalInterface + interface DataTieringMethodCallerWithKey { + boolean call(DataTieringManager manager, BlockCacheKey key) throws DataTieringException; + } + + @Test + public void testDataTieringEnabledWithKey() { + DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isDataTieringEnabled; + + // Test with valid key + BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true); + + // Test with another valid key + key = new BlockCacheKey(hStoreFiles.get(1).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false); + + // Test with valid key with no HFile Path + key = new BlockCacheKey(hStoreFiles.get(0).getPath().getName(), 0); + testDataTieringMethodWithKeyExpectingException(methodCallerWithKey, key, + new DataTieringException("BlockCacheKey Doesn't Contain HFile Path")); + } + + @Test + public void testDataTieringEnabledWithPath() { + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isDataTieringEnabled; + + // Test with valid path + Path hFilePath = hStoreFiles.get(1).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Test with another valid path + hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + + // Test with an incorrect path + hFilePath = new Path("incorrectPath"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("Incorrect HFile Path: " + hFilePath)); + + // Test with a non-existing HRegion path + Path basePath = hStoreFiles.get(0).getPath().getParent().getParent().getParent(); + hFilePath = new Path(basePath, "incorrectRegion/cf1/filename"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist")); + + // Test with a non-existing HStore path + basePath = hStoreFiles.get(0).getPath().getParent().getParent(); + hFilePath = new Path(basePath, "incorrectCf/filename"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist")); + } + + @Test + public void testHotDataWithKey() { + DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isHotData; + + // Test with valid key + BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true); + + // Test with another valid key + key = new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false); + } + + @Test + public void testHotDataWithPath() { + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData; + + // Test with valid path + Path hFilePath = hStoreFiles.get(2).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + + // Test with another valid path + hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Test with a filename where corresponding HStoreFile in not present + hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName"); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + } + + @Test + public void testColdDataFiles() { + Set allCachedBlocks = new HashSet<>(); + for (HStoreFile file : hStoreFiles) { + allCachedBlocks.add(new BlockCacheKey(file.getPath(), 0, true, BlockType.DATA)); + } + + // Verify hStoreFile3 is identified as cold data + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData; + Path hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Verify all the other files in hStoreFiles are hot data + for (int i = 0; i < hStoreFiles.size() - 1; i++) { + hFilePath = hStoreFiles.get(i).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + } + + try { + Set coldFilePaths = dataTieringManager.getColdDataFiles(allCachedBlocks); + assertEquals(1, coldFilePaths.size()); + } catch (DataTieringException e) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + } + + private void testDataTieringMethodWithPath(DataTieringMethodCallerWithPath caller, Path path, + boolean expectedResult, DataTieringException exception) { + try { + boolean value = caller.call(dataTieringManager, path); + if (exception != null) { + fail("Expected DataTieringException to be thrown"); + } + assertEquals(expectedResult, value); + } catch (DataTieringException e) { + if (exception == null) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + assertEquals(exception.getMessage(), e.getMessage()); + } + } + + private void testDataTieringMethodWithKey(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, boolean expectedResult, DataTieringException exception) { + try { + boolean value = caller.call(dataTieringManager, key); + if (exception != null) { + fail("Expected DataTieringException to be thrown"); + } + assertEquals(expectedResult, value); + } catch (DataTieringException e) { + if (exception == null) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + assertEquals(exception.getMessage(), e.getMessage()); + } + } + + private void testDataTieringMethodWithPathExpectingException( + DataTieringMethodCallerWithPath caller, Path path, DataTieringException exception) { + testDataTieringMethodWithPath(caller, path, false, exception); + } + + private void testDataTieringMethodWithPathNoException(DataTieringMethodCallerWithPath caller, + Path path, boolean expectedResult) { + testDataTieringMethodWithPath(caller, path, expectedResult, null); + } + + private void testDataTieringMethodWithKeyExpectingException(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, DataTieringException exception) { + testDataTieringMethodWithKey(caller, key, false, exception); + } + + private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, boolean expectedResult) { + testDataTieringMethodWithKey(caller, key, expectedResult, null); + } + + private static void setupOnlineRegions() throws IOException { + testOnlineRegions = new HashMap<>(); + hStoreFiles = new ArrayList<>(); + + long day = 24 * 60 * 60 * 1000; + long currentTime = System.currentTimeMillis(); + + HRegion region1 = createHRegion("table1"); + + HStore hStore11 = createHStore(region1, "cf1", getConfWithTimeRangeDataTieringEnabled(day)); + hStoreFiles + .add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), currentTime)); + hStore11.refreshStoreFiles(); + HStore hStore12 = createHStore(region1, "cf2"); + hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(), + currentTime - day)); + hStore12.refreshStoreFiles(); + + region1.stores.put(Bytes.toBytes("cf1"), hStore11); + region1.stores.put(Bytes.toBytes("cf2"), hStore12); + + HRegion region2 = + createHRegion("table2", getConfWithTimeRangeDataTieringEnabled((long) (2.5 * day))); + + HStore hStore21 = createHStore(region2, "cf1"); + hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(), + currentTime - 2 * day)); + hStore21.refreshStoreFiles(); + HStore hStore22 = createHStore(region2, "cf2"); + hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(), + currentTime - 3 * day)); + hStore22.refreshStoreFiles(); + + region2.stores.put(Bytes.toBytes("cf1"), hStore21); + region2.stores.put(Bytes.toBytes("cf2"), hStore22); + + for (HStoreFile file : hStoreFiles) { + file.initReader(); + } + + testOnlineRegions.put(region1.getRegionInfo().getEncodedName(), region1); + testOnlineRegions.put(region2.getRegionInfo().getEncodedName(), region2); + } + + private static HRegion createHRegion(String table) throws IOException { + return createHRegion(table, defaultConf); + } + + private static HRegion createHRegion(String table, Configuration conf) throws IOException { + TableName tableName = TableName.valueOf(table); + + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) + .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, + conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .build(); + RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); + + Configuration testConf = new Configuration(conf); + CommonFSUtils.setRootDir(testConf, testDir); + HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); + + return new HRegion(regionFs, null, conf, htd, null); + } + + private static HStore createHStore(HRegion region, String columnFamily) throws IOException { + return createHStore(region, columnFamily, defaultConf); + } + + private static HStore createHStore(HRegion region, String columnFamily, Configuration conf) + throws IOException { + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)) + .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) + .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, + conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .build(); + + return new HStore(region, columnFamilyDescriptor, conf, false); + } + + private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotDataAge) { + Configuration conf = new Configuration(defaultConf); + conf.set(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name()); + conf.set(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(hotDataAge)); + return conf; + } + + private static HStoreFile createHStoreFile(Path storeDir, long timestamp) throws IOException { + String columnFamily = storeDir.getName(); + + StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(defaultConf, cacheConf, fs) + .withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build(); + + writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), Bytes.toBytes("random"), + timestamp); + + return new HStoreFile(fs, storeFileWriter.getPath(), defaultConf, cacheConf, BloomType.NONE, + true); + } + + private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily, + byte[] qualifier, long timestamp) throws IOException { + try { + for (char d = 'a'; d <= 'z'; d++) { + for (char e = 'a'; e <= 'z'; e++) { + byte[] b = new byte[] { (byte) d, (byte) e }; + writer.append(new KeyValue(b, columnFamily, qualifier, timestamp, b)); + } + } + } finally { + writer.appendTrackedTimestampsToMetadata(); + writer.close(); + } + } +} From 063e4cc5682c78754d472e6c6dbf85020a6a0749 Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Fri, 12 Apr 2024 14:54:37 +0530 Subject: [PATCH 041/336] HBASE-28505 Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering (#5809) Signed-off-by: Wellington Chevreuil Change-Id: I30772e5e4ea0e91f862327616a108bd1033fee89 --- .../regionserver/DataTieringManager.java | 2 +- .../regionserver/DateTieredStoreEngine.java | 3 + .../hbase/util/TableDescriptorChecker.java | 36 ++++++++ .../client/TestIllegalTableDescriptor.java | 39 ++++++++ .../regionserver/TestDataTieringManager.java | 89 +++++++++++++------ 5 files changed, 142 insertions(+), 27 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 0bc04ddc428b..2903963f706e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -199,7 +199,7 @@ private HStore getHStore(Path hFilePath) throws DataTieringException { private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException { HStore hStore = getHStore(hFilePath); for (HStoreFile file : hStore.getStorefiles()) { - if (file.getPath().equals(hFilePath)) { + if (file.getPath().toUri().getPath().toString().equals(hFilePath.toString())) { return file; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index ded6564bce53..26437ab11242 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -41,6 +41,9 @@ @InterfaceAudience.Private public class DateTieredStoreEngine extends StoreEngine { + + public static final String DATE_TIERED_STORE_ENGINE = DateTieredStoreEngine.class.getName(); + @Override public boolean needsCompaction(List filesCompacting) { return compactionPolicy.needsCompaction(storeFileManager.getStoreFiles(), filesCompacting); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java index a826860aae41..409cc284dee2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.util; +import static org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine.DATE_TIERED_STORE_ENGINE; + import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompoundConfiguration; @@ -29,10 +31,13 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.conf.ConfigKey; import org.apache.hadoop.hbase.fs.ErasureCodingUtils; +import org.apache.hadoop.hbase.regionserver.DataTieringManager; +import org.apache.hadoop.hbase.regionserver.DataTieringType; import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; import org.apache.yetus.audience.InterfaceAudience; @@ -201,6 +206,8 @@ public static void sanityCheck(final Configuration c, final TableDescriptor td) // check in-memory compaction warnOrThrowExceptionForFailure(logWarn, hcd::getInMemoryCompaction); + + checkDateTieredCompactionForTimeRangeDataTiering(conf, td); } } @@ -220,6 +227,35 @@ private static void checkReplicationScope(final Configuration conf, final TableD }); } + private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf, + final TableDescriptor td) throws IOException { + // Table level configurations + checkDateTieredCompactionForTimeRangeDataTiering(conf); + for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) { + // Column family level configurations + Configuration cfdConf = + new CompoundConfiguration().add(conf).addStringMap(cfd.getConfiguration()); + checkDateTieredCompactionForTimeRangeDataTiering(cfdConf); + } + } + + private static void checkDateTieredCompactionForTimeRangeDataTiering(final Configuration conf) + throws IOException { + final String errorMessage = + "Time Range Data Tiering should be enabled with Date Tiered Compaction."; + + warnOrThrowExceptionForFailure(false, () -> { + + // Determine whether Date Tiered Compaction will be enabled when Time Range Data Tiering is + // enabled after the configuration change. + if (DataTieringType.TIME_RANGE.name().equals(conf.get(DataTieringManager.DATATIERING_KEY))) { + if (!DATE_TIERED_STORE_ENGINE.equals(conf.get(StoreEngine.STORE_ENGINE_CLASS_KEY))) { + throw new IllegalArgumentException(errorMessage); + } + } + }); + } + private static void checkCompactionPolicy(final Configuration conf, final TableDescriptor td) throws IOException { warnOrThrowExceptionForFailure(false, () -> { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java index c566432b4e76..2d45c05324be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java @@ -33,6 +33,9 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.DataTieringManager; +import org.apache.hadoop.hbase.regionserver.DataTieringType; +import org.apache.hadoop.hbase.regionserver.StoreEngine; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -189,6 +192,42 @@ public void testIllegalTableDescriptor() throws Exception { + "cause very frequent flushing.")); } + @Test + public void testIllegalTableDescriptorWithDataTiering() throws IOException { + // table level configuration changes + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + + // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine + htd.setValue(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name()); + checkTableIsIllegal(htd); + + // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine + htd.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY, + "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine"); + checkTableIsLegal(htd); + + // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active + htd.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY, + "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine"); + checkTableIsIllegal(htd); + + // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine + hcd.setConfiguration(DataTieringManager.DATATIERING_KEY, + DataTieringType.TIME_RANGE.name()); + checkTableIsIllegal(htd.addFamily(hcd)); + + // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine + hcd.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY, + "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine"); + checkTableIsLegal(htd.addFamily(hcd)); + + // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active + hcd.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY, + "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine"); + checkTableIsIllegal(htd.addFamily(hcd)); + } + private void checkTableIsLegal(HTableDescriptor htd) throws IOException { Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index afb5862a8a46..548539445836 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -19,19 +19,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; - import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -47,6 +47,8 @@ import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -82,15 +84,21 @@ public class TestDataTieringManager { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestDataTieringManager.class); - private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static Configuration defaultConf; private static FileSystem fs; private static CacheConfig cacheConf; private static Path testDir; - private static Map testOnlineRegions; - + private static final Map testOnlineRegions = new HashMap<>(); private static DataTieringManager dataTieringManager; - private static List hStoreFiles; + private static final List hStoreFiles = new ArrayList<>(); + + /** + * Represents the current lexicographically increasing string used as a row key when writing + * HFiles. It is incremented each time {@link #nextString()} is called to generate unique row + * keys. + */ + private static String rowKeyString; @BeforeClass public static void setupBeforeClass() throws Exception { @@ -271,21 +279,20 @@ private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWith } private static void setupOnlineRegions() throws IOException { - testOnlineRegions = new HashMap<>(); - hStoreFiles = new ArrayList<>(); - + testOnlineRegions.clear(); + hStoreFiles.clear(); long day = 24 * 60 * 60 * 1000; long currentTime = System.currentTimeMillis(); HRegion region1 = createHRegion("table1"); HStore hStore11 = createHStore(region1, "cf1", getConfWithTimeRangeDataTieringEnabled(day)); - hStoreFiles - .add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), currentTime)); + hStoreFiles.add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), + hStore11.getReadOnlyConfiguration(), currentTime, region1.getRegionFileSystem())); hStore11.refreshStoreFiles(); HStore hStore12 = createHStore(region1, "cf2"); hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(), - currentTime - day)); + hStore12.getReadOnlyConfiguration(), currentTime - day, region1.getRegionFileSystem())); hStore12.refreshStoreFiles(); region1.stores.put(Bytes.toBytes("cf1"), hStore11); @@ -296,11 +303,11 @@ private static void setupOnlineRegions() throws IOException { HStore hStore21 = createHStore(region2, "cf1"); hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(), - currentTime - 2 * day)); + hStore21.getReadOnlyConfiguration(), currentTime - 2 * day, region2.getRegionFileSystem())); hStore21.refreshStoreFiles(); HStore hStore22 = createHStore(region2, "cf2"); hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(), - currentTime - 3 * day)); + hStore22.getReadOnlyConfiguration(), currentTime - 3 * day, region2.getRegionFileSystem())); hStore22.refreshStoreFiles(); region2.stores.put(Bytes.toBytes("cf1"), hStore21); @@ -359,31 +366,61 @@ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotData return conf; } - private static HStoreFile createHStoreFile(Path storeDir, long timestamp) throws IOException { + + static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp, + HRegionFileSystem regionFs) throws IOException { String columnFamily = storeDir.getName(); - StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(defaultConf, cacheConf, fs) + StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(conf, cacheConf, fs) .withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build(); - writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), Bytes.toBytes("random"), - timestamp); + writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); + + StoreContext storeContext = StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); - return new HStoreFile(fs, storeFileWriter.getPath(), defaultConf, cacheConf, BloomType.NONE, - true); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, + sft); } private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily, - byte[] qualifier, long timestamp) throws IOException { + long timestamp) throws IOException { + int cellsPerFile = 10; + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] value = generateRandomBytes(4 * 1024); try { - for (char d = 'a'; d <= 'z'; d++) { - for (char e = 'a'; e <= 'z'; e++) { - byte[] b = new byte[] { (byte) d, (byte) e }; - writer.append(new KeyValue(b, columnFamily, qualifier, timestamp, b)); - } + for (int i = 0; i < cellsPerFile; i++) { + byte[] row = Bytes.toBytes(nextString()); + writer.append(new KeyValue(row, columnFamily, qualifier, timestamp, value)); } } finally { writer.appendTrackedTimestampsToMetadata(); writer.close(); } } + + + private static byte[] generateRandomBytes(int sizeInBytes) { + Random random = new Random(); + byte[] randomBytes = new byte[sizeInBytes]; + random.nextBytes(randomBytes); + return randomBytes; + } + + /** + * Returns the lexicographically larger string every time it's called. + */ + private static String nextString() { + if (rowKeyString == null || rowKeyString.isEmpty()) { + rowKeyString = "a"; + } + char lastChar = rowKeyString.charAt(rowKeyString.length() - 1); + if (lastChar < 'z') { + rowKeyString = rowKeyString.substring(0, rowKeyString.length() - 1) + (char) (lastChar + 1); + } else { + rowKeyString = rowKeyString + "a"; + } + return rowKeyString; + } + } From 3e114641e54d10c78ca5d3d7179c1bca4e73de2c Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Mon, 22 Apr 2024 15:23:30 +0530 Subject: [PATCH 042/336] HBASE-28466 Integration of time-based priority logic of bucket cache in prefetch functionality of HBase (#5808) Signed-off-by: Wellington Chevreuil Change-Id: I374a9ab88807da3188962d7ac0b2b727eaffd010 --- .../hadoop/hbase/io/hfile/BlockCache.java | 5 +- .../hbase/io/hfile/CombinedBlockCache.java | 6 +- .../hadoop/hbase/io/hfile/HFileInfo.java | 6 ++ .../hbase/io/hfile/HFilePreadReader.java | 2 + .../hbase/io/hfile/bucket/BucketCache.java | 15 ++- .../regionserver/DataTieringManager.java | 91 +++++++++++++++---- .../regionserver/TestDataTieringManager.java | 57 ++++++++++-- 7 files changed, 150 insertions(+), 32 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 8f12e367e588..8380fc194e7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -184,11 +184,12 @@ default Optional blockFitsIntoTheCache(HFileBlock block) { * overridden by all implementing classes. In such cases, the returned Optional will be empty. For * subclasses implementing this logic, the returned Optional would contain the boolean value * reflecting if the passed file should indeed be cached. - * @param fileName to check if it should be cached. + * @param hFileInfo Information about the file to check if it should be cached. + * @param conf The configuration object to use for determining caching behavior. * @return empty optional if this method is not supported, otherwise the returned optional * contains the boolean value informing if the file should be cached. */ - default Optional shouldCacheFile(String fileName) { + default Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf) { return Optional.empty(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 61976a86f83e..fe675aade7bb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -477,9 +477,9 @@ public Optional blockFitsIntoTheCache(HFileBlock block) { } @Override - public Optional shouldCacheFile(String fileName) { - Optional l1Result = l1Cache.shouldCacheFile(fileName); - Optional l2Result = l2Cache.shouldCacheFile(fileName); + public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf) { + Optional l1Result = l1Cache.shouldCacheFile(hFileInfo, conf); + Optional l2Result = l2Cache.shouldCacheFile(hFileInfo, conf); final Mutable combinedResult = new MutableBoolean(true); l1Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); l2Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java index e16235373856..b3ccd2487800 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java @@ -121,6 +121,7 @@ public class HFileInfo implements SortedMap { private FixedFileTrailer trailer; private HFileContext hfileContext; + private boolean initialized = false; public HFileInfo() { super(); @@ -361,6 +362,10 @@ public void initTrailerAndContext(ReaderContext context, Configuration conf) thr * should be called after initTrailerAndContext */ public void initMetaAndIndex(HFile.Reader reader) throws IOException { + if (initialized) { + return; + } + ReaderContext context = reader.getContext(); try { HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); @@ -398,6 +403,7 @@ public void initMetaAndIndex(HFile.Reader reader) throws IOException { throw new CorruptHFileException( "Problem reading data index and meta index from file " + context.getFilePath(), t); } + initialized = true; } private HFileContext createHFileContext(Path path, FixedFileTrailer trailer, Configuration conf) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index 08649ebb3156..3ef5f50db029 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -35,6 +35,8 @@ public class HFilePreadReader extends HFileReaderImpl { public HFilePreadReader(ReaderContext context, HFileInfo fileInfo, CacheConfig cacheConf, Configuration conf) throws IOException { super(context, fileInfo, cacheConf, conf); + // Initialize HFileInfo object with metadata for caching decisions + fileInfo.initMetaAndIndex(this); // master hosted regions, like the master procedures store wouldn't have a block cache // Prefetch file blocks upon open if requested if (cacheConf.getBlockCache().isPresent() && cacheConf.shouldPrefetchOnOpen()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 28b2b05936cf..569a04bda8f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -79,9 +79,11 @@ import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.RefCnt; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; +import org.apache.hadoop.hbase.regionserver.DataTieringManager; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.Bytes; @@ -2317,7 +2319,18 @@ public Optional blockFitsIntoTheCache(HFileBlock block) { } @Override - public Optional shouldCacheFile(String fileName) { + public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf) { + String fileName = hFileInfo.getHFileContext().getHFileName(); + try { + DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + if (!dataTieringManager.isHotData(hFileInfo, conf)) { + LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", fileName); + return Optional.of(false); + } + } catch (IllegalStateException e) { + LOG.error("Error while getting DataTieringManager instance: {}", e.getMessage()); + } + // if we don't have the file in fullyCachedFiles, we should cache it return Optional.of(!fullyCachedFiles.containsKey(fileName)); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 2903963f706e..b781052efa9e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; + +import java.io.IOException; import java.util.HashSet; import java.util.Map; import java.util.OptionalLong; @@ -24,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; @@ -106,11 +110,13 @@ public boolean isDataTieringEnabled(Path hFilePath) throws DataTieringException } /** - * Determines whether the data associated with the given block cache key is considered hot. + * Determines whether the data associated with the given block cache key is considered hot. If the + * data tiering type is set to {@link DataTieringType#TIME_RANGE} and maximum timestamp is not + * present, it considers {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by + * default. * @param key the block cache key * @return {@code true} if the data is hot, {@code false} otherwise - * @throws DataTieringException if there is an error retrieving data tiering information or the - * HFile maximum timestamp + * @throws DataTieringException if there is an error retrieving data tiering information */ public boolean isHotData(BlockCacheKey key) throws DataTieringException { Path hFilePath = key.getFilePath(); @@ -122,37 +128,82 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException { /** * Determines whether the data in the HFile at the given path is considered hot based on the - * configured data tiering type and hot data age. + * configured data tiering type and hot data age. If the data tiering type is set to + * {@link DataTieringType#TIME_RANGE} and maximum timestamp is not present, it considers + * {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by default. * @param hFilePath the path to the HFile * @return {@code true} if the data is hot, {@code false} otherwise - * @throws DataTieringException if there is an error retrieving data tiering information or the - * HFile maximum timestamp + * @throws DataTieringException if there is an error retrieving data tiering information */ public boolean isHotData(Path hFilePath) throws DataTieringException { Configuration configuration = getConfiguration(hFilePath); DataTieringType dataTieringType = getDataTieringType(configuration); if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { - long hotDataAge = getDataTieringHotDataAge(configuration); - - HStoreFile hStoreFile = getHStoreFile(hFilePath); - if (hStoreFile == null) { - LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist"); - return false; - } - OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); - if (!maxTimestamp.isPresent()) { - throw new DataTieringException("Maximum timestamp not present for " + hFilePath); - } + return hotDataValidator(getMaxTimestamp(hFilePath), getDataTieringHotDataAge(configuration)); + } + // DataTieringType.NONE or other types are considered hot by default + return true; + } - long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime(); - long diff = currentTimestamp - maxTimestamp.getAsLong(); - return diff <= hotDataAge; + /** + * Determines whether the data in the HFile being read is considered hot based on the configured + * data tiering type and hot data age. If the data tiering type is set to + * {@link DataTieringType#TIME_RANGE} and maximum timestamp is not present, it considers + * {@code Long.MAX_VALUE} as the maximum timestamp, making the data hot by default. + * @param hFileInfo Information about the HFile to determine if its data is hot. + * @param configuration The configuration object to use for determining hot data criteria. + * @return {@code true} if the data is hot, {@code false} otherwise + */ + public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) { + DataTieringType dataTieringType = getDataTieringType(configuration); + if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { + return hotDataValidator(getMaxTimestamp(hFileInfo), getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; } + private boolean hotDataValidator(long maxTimestamp, long hotDataAge) { + long currentTimestamp = getCurrentTimestamp(); + long diff = currentTimestamp - maxTimestamp; + return diff <= hotDataAge; + } + + private long getMaxTimestamp(Path hFilePath) throws DataTieringException { + HStoreFile hStoreFile = getHStoreFile(hFilePath); + if (hStoreFile == null) { + LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist"); + return Long.MAX_VALUE; + } + OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); + if (!maxTimestamp.isPresent()) { + LOG.error("Maximum timestamp not present for " + hFilePath); + return Long.MAX_VALUE; + } + return maxTimestamp.getAsLong(); + } + + private long getMaxTimestamp(HFileInfo hFileInfo) { + try { + byte[] hFileTimeRange = hFileInfo.get(TIMERANGE_KEY); + if (hFileTimeRange == null) { + LOG.error("Timestamp information not found for file: {}", + hFileInfo.getHFileContext().getHFileName()); + return Long.MAX_VALUE; + } + return TimeRangeTracker.parseFrom(hFileTimeRange).getMax(); + } catch (IOException e) { + LOG.error("Error occurred while reading the timestamp metadata of file: {}", + hFileInfo.getHFileContext().getHFileName(), e); + return Long.MAX_VALUE; + } + } + + private long getCurrentTimestamp() { + return EnvironmentEdgeManager.getDelegate().currentTime(); + } + /** * Returns a set of cold data filenames from the given set of cached blocks. Cold data is * determined by the configured data tiering type and hot data age. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 548539445836..3e99d453d5b7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; @@ -26,14 +28,17 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Optional; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -53,6 +58,7 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.Pair; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -87,9 +93,11 @@ public class TestDataTieringManager { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static Configuration defaultConf; private static FileSystem fs; + private static BlockCache blockCache; private static CacheConfig cacheConf; private static Path testDir; private static final Map testOnlineRegions = new HashMap<>(); + private static DataTieringManager dataTieringManager; private static final List hStoreFiles = new ArrayList<>(); @@ -104,11 +112,14 @@ public class TestDataTieringManager { public static void setupBeforeClass() throws Exception { testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName()); defaultConf = TEST_UTIL.getConfiguration(); + defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); + defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32); fs = HFileSystem.get(defaultConf); - BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf); + blockCache = BlockCacheFactory.createBlockCache(defaultConf); cacheConf = new CacheConfig(defaultConf, blockCache); - setupOnlineRegions(); DataTieringManager.instantiate(testOnlineRegions); + setupOnlineRegions(); dataTieringManager = DataTieringManager.getInstance(); } @@ -197,7 +208,30 @@ public void testHotDataWithPath() { // Test with a filename where corresponding HStoreFile in not present hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName"); - testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + } + + @Test + public void testPrefetchWhenDataTieringEnabled() throws IOException { + setPrefetchBlocksOnOpen(); + initializeTestEnvironment(); + // Evict blocks from cache by closing the files and passing evict on close. + // Then initialize the reader again. Since Prefetch on open is set to true, it should prefetch + // those blocks. + for (HStoreFile file : hStoreFiles) { + file.closeStoreFile(true); + file.initReader(); + } + + // Since we have one cold file among four files, only three should get prefetched. + Optional>> fullyCachedFiles = blockCache.getFullyCachedFiles(); + assertTrue("We should get the fully cached files from the cache", fullyCachedFiles.isPresent()); + Waiter.waitFor(defaultConf, 10000, () -> fullyCachedFiles.get().size() == 3); + assertEquals("Number of fully cached files are incorrect", 3, fullyCachedFiles.get().size()); + } + + private void setPrefetchBlocksOnOpen() { + defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); } @Test @@ -278,6 +312,17 @@ private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWith testDataTieringMethodWithKey(caller, key, expectedResult, null); } + private static void initializeTestEnvironment() throws IOException { + setupFileSystemAndCache(); + setupOnlineRegions(); + } + + private static void setupFileSystemAndCache() throws IOException { + fs = HFileSystem.get(defaultConf); + blockCache = BlockCacheFactory.createBlockCache(defaultConf); + cacheConf = new CacheConfig(defaultConf, blockCache); + } + private static void setupOnlineRegions() throws IOException { testOnlineRegions.clear(); hStoreFiles.clear(); @@ -376,11 +421,11 @@ static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long times writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); - StoreContext storeContext = StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); + StoreContext storeContext = + StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); - return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, - sft); + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, sft); } private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily, From b28216138458b3a914bfb0b4e82d69898428f19f Mon Sep 17 00:00:00 2001 From: jhungund <106576553+jhungund@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:29:36 +0530 Subject: [PATCH 043/336] HBASE-28468: Integrate the data-tiering logic into cache evictions. (#5829) Signed-off-by: Wellington Chevreuil --- .../hbase/io/hfile/bucket/BucketCache.java | 36 +++- .../regionserver/DataTieringManager.java | 42 ++++- .../regionserver/TestDataTieringManager.java | 178 ++++++++++++++++++ 3 files changed, 253 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 569a04bda8f8..1260c0b6b5ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -1075,6 +1075,14 @@ void freeSpace(final String why) { // the cached time is recored in nanos, so we need to convert the grace period accordingly long orphanGracePeriodNanos = orphanBlockGracePeriod * 1000000; long bytesFreed = 0; + // Check the list of files to determine the cold files which can be readily evicted. + Map coldFiles = null; + try { + DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + coldFiles = dataTieringManager.getColdFilesList(); + } catch (IllegalStateException e) { + LOG.warn("Data Tiering Manager is not set. Ignore time-based block evictions."); + } // Scan entire map putting bucket entry into appropriate bucket entry // group for (Map.Entry bucketEntryWithKey : backingMap.entrySet()) { @@ -1107,6 +1115,17 @@ void freeSpace(final String why) { } } } + + if (bytesFreed < bytesToFreeWithExtra && + coldFiles != null && coldFiles.containsKey(bucketEntryWithKey.getKey().getHfileName()) + ) { + int freedBlockSize = bucketEntryWithKey.getValue().getLength(); + if (evictBlockIfNoRpcReferenced(bucketEntryWithKey.getKey())) { + bytesFreed += freedBlockSize; + } + continue; + } + switch (entry.getPriority()) { case SINGLE: { bucketSingle.add(bucketEntryWithKey); @@ -1122,6 +1141,22 @@ void freeSpace(final String why) { } } } + + // Check if the cold file eviction is sufficient to create enough space. + bytesToFreeWithExtra -= bytesFreed; + if (bytesToFreeWithExtra <= 0) { + LOG.debug("Bucket cache free space completed; freed space : {} bytes of cold data blocks.", + StringUtils.byteDesc(bytesFreed)); + return; + } + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Bucket cache free space completed; freed space : {} " + + "bytes of cold data blocks. {} more bytes required to be freed.", + StringUtils.byteDesc(bytesFreed), bytesToFreeWithExtra); + } + PriorityQueue bucketQueue = new PriorityQueue<>(3, Comparator.comparingLong(BucketEntryGroup::overflow)); @@ -1130,7 +1165,6 @@ void freeSpace(final String why) { bucketQueue.add(bucketMemory); int remainingBuckets = bucketQueue.size(); - BucketEntryGroup bucketGroup; while ((bucketGroup = bucketQueue.poll()) != null) { long overflow = bucketGroup.overflow(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index b781052efa9e..a1bc26603805 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.OptionalLong; @@ -173,12 +174,12 @@ private boolean hotDataValidator(long maxTimestamp, long hotDataAge) { private long getMaxTimestamp(Path hFilePath) throws DataTieringException { HStoreFile hStoreFile = getHStoreFile(hFilePath); if (hStoreFile == null) { - LOG.error("HStoreFile corresponding to " + hFilePath + " doesn't exist"); + LOG.error("HStoreFile corresponding to {} doesn't exist", hFilePath); return Long.MAX_VALUE; } OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); if (!maxTimestamp.isPresent()) { - LOG.error("Maximum timestamp not present for " + hFilePath); + LOG.error("Maximum timestamp not present for {}", hFilePath); return Long.MAX_VALUE; } return maxTimestamp.getAsLong(); @@ -270,4 +271,41 @@ private long getDataTieringHotDataAge(Configuration conf) { return Long.parseLong( conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE))); } + + /* + * This API traverses through the list of online regions and returns a subset of these files-names + * that are cold. + * @return List of names of files with cold data as per data-tiering logic. + */ + public Map getColdFilesList() { + Map coldFiles = new HashMap<>(); + for (HRegion r : this.onlineRegions.values()) { + for (HStore hStore : r.getStores()) { + Configuration conf = hStore.getReadOnlyConfiguration(); + if (getDataTieringType(conf) != DataTieringType.TIME_RANGE) { + // Data-Tiering not enabled for the store. Just skip it. + continue; + } + Long hotDataAge = getDataTieringHotDataAge(conf); + + for (HStoreFile hStoreFile : hStore.getStorefiles()) { + String hFileName = + hStoreFile.getFileInfo().getHFileInfo().getHFileContext().getHFileName(); + OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); + if (!maxTimestamp.isPresent()) { + LOG.warn("maxTimestamp missing for file: {}", + hStoreFile.getFileInfo().getActiveFileName()); + continue; + } + long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime(); + long fileAge = currentTimestamp - maxTimestamp.getAsLong(); + if (fileAge > hotDataAge) { + // Values do not matter. + coldFiles.put(hFileName, null); + } + } + } + } + return coldFiles; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 3e99d453d5b7..9c8073961b8b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; +import static org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.DEFAULT_ERROR_TOLERATION_DURATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -51,7 +52,9 @@ import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -260,6 +263,181 @@ public void testColdDataFiles() { } } + @Test + public void testPickColdDataFiles() { + Map coldDataFiles = dataTieringManager.getColdFilesList(); + assertEquals(1, coldDataFiles.size()); + // hStoreFiles[3] is the cold file. + assert (coldDataFiles.containsKey(hStoreFiles.get(3).getFileInfo().getActiveFileName())); + } + + /* + * Verify that two cold blocks(both) are evicted when bucket reaches its capacity. The hot file + * remains in the cache. + */ + @Test + public void testBlockEvictions() throws Exception { + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, + 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", + DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with cold data files and a block with hot data. + // hStoreFiles.get(3) is a cold data file, while hStoreFiles.get(0) is a hot file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block into cache with hot data which should trigger the eviction + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 2 hot blocks blocks only. + // Both cold blocks of 8KB will be evicted to make room for 1 block of 8KB + an additional + // space. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0); + } + + /* + * Verify that two cold blocks(both) are evicted when bucket reaches its capacity, but one cold + * block remains in the cache since the required space is freed. + */ + @Test + public void testBlockEvictionsAllColdBlocks() throws Exception { + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, + 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", + DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with three cold data blocks. + // hStoreFiles.get(3) is a cold data file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 16384, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block into cache with hot data which should trigger the eviction + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 1 cold block and a newly added hot block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 1, 1); + } + + /* + * Verify that a hot block evicted along with a cold block when bucket reaches its capacity. + */ + @Test + public void testBlockEvictionsHotBlocks() throws Exception { + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, + 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", + DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with two hot data blocks and one cold data block + // hStoreFiles.get(0) is a hot data file and hStoreFiles.get(3) is a cold data file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block which should evict the only cold block with an additional hot block. + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 2 hot blocks. + // Only one of the older hot blocks is retained and other one is the newly added hot block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0); + } + + private void validateBlocks(Set keys, int expectedTotalKeys, int expectedHotBlocks, + int expectedColdBlocks) { + int numHotBlocks = 0, numColdBlocks = 0; + + assertEquals(expectedTotalKeys, keys.size()); + int iter = 0; + for (BlockCacheKey key : keys) { + try { + if (dataTieringManager.isHotData(key)) { + numHotBlocks++; + } else { + numColdBlocks++; + } + } catch (Exception e) { + fail("Unexpected exception!"); + } + } + assertEquals(expectedHotBlocks, numHotBlocks); + assertEquals(expectedColdBlocks, numColdBlocks); + } + private void testDataTieringMethodWithPath(DataTieringMethodCallerWithPath caller, Path path, boolean expectedResult, DataTieringException exception) { try { From aaab14e18bf749ab8e43c25599315736c55fc933 Mon Sep 17 00:00:00 2001 From: jhungund <106576553+jhungund@users.noreply.github.com> Date: Thu, 2 May 2024 13:54:33 +0530 Subject: [PATCH 044/336] HBASE-28535: Add a region-server wide key to enable data-tiering. (#5856) Signed-off-by: Wellington Chevreuil Change-Id: Iace49e6f57b15ebe44ab12591ed72be1d20e0391 --- .../hbase/io/hfile/bucket/BucketCache.java | 20 ++-- .../regionserver/DataTieringManager.java | 32 +++++-- .../hbase/regionserver/HRegionServer.java | 4 +- .../regionserver/TestDataTieringManager.java | 91 ++++++++++++++++--- 4 files changed, 113 insertions(+), 34 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 1260c0b6b5ee..7242e40bf05a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -1077,11 +1077,10 @@ void freeSpace(final String why) { long bytesFreed = 0; // Check the list of files to determine the cold files which can be readily evicted. Map coldFiles = null; - try { - DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + + DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + if (dataTieringManager != null) { coldFiles = dataTieringManager.getColdFilesList(); - } catch (IllegalStateException e) { - LOG.warn("Data Tiering Manager is not set. Ignore time-based block evictions."); } // Scan entire map putting bucket entry into appropriate bucket entry // group @@ -2355,16 +2354,11 @@ public Optional blockFitsIntoTheCache(HFileBlock block) { @Override public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf) { String fileName = hFileInfo.getHFileContext().getHFileName(); - try { - DataTieringManager dataTieringManager = DataTieringManager.getInstance(); - if (!dataTieringManager.isHotData(hFileInfo, conf)) { - LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", fileName); - return Optional.of(false); - } - } catch (IllegalStateException e) { - LOG.error("Error while getting DataTieringManager instance: {}", e.getMessage()); + DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + if (dataTieringManager != null && !dataTieringManager.isHotData(hFileInfo, conf)) { + LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", fileName); + return Optional.of(false); } - // if we don't have the file in fullyCachedFiles, we should cache it return Optional.of(!fullyCachedFiles.containsKey(fileName)); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index a1bc26603805..d3bdbf330cc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -45,6 +45,9 @@ @InterfaceAudience.Private public class DataTieringManager { private static final Logger LOG = LoggerFactory.getLogger(DataTieringManager.class); + public static final String GLOBAL_DATA_TIERING_ENABLED_KEY = + "hbase.regionserver.datatiering.enable"; + public static final boolean DEFAULT_GLOBAL_DATA_TIERING_ENABLED = false; // disabled by default public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type"; public static final String DATATIERING_HOT_DATA_AGE_KEY = "hbase.hstore.datatiering.hot.age.millis"; @@ -58,28 +61,29 @@ private DataTieringManager(Map onlineRegions) { } /** - * Initializes the DataTieringManager instance with the provided map of online regions. + * Initializes the DataTieringManager instance with the provided map of online regions, only if + * the configuration "hbase.regionserver.datatiering.enable" is enabled. + * @param conf Configuration object. * @param onlineRegions A map containing online regions. + * @return True if the instance is instantiated successfully, false otherwise. */ - public static synchronized void instantiate(Map onlineRegions) { - if (instance == null) { + public static synchronized boolean instantiate(Configuration conf, + Map onlineRegions) { + if (isDataTieringFeatureEnabled(conf) && instance == null) { instance = new DataTieringManager(onlineRegions); LOG.info("DataTieringManager instantiated successfully."); + return true; } else { LOG.warn("DataTieringManager is already instantiated."); } + return false; } /** * Retrieves the instance of DataTieringManager. - * @return The instance of DataTieringManager. - * @throws IllegalStateException if DataTieringManager has not been instantiated. + * @return The instance of DataTieringManager, if instantiated, null otherwise. */ public static synchronized DataTieringManager getInstance() { - if (instance == null) { - throw new IllegalStateException( - "DataTieringManager has not been instantiated. Call instantiate() first."); - } return instance; } @@ -308,4 +312,14 @@ public Map getColdFilesList() { } return coldFiles; } + + private static boolean isDataTieringFeatureEnabled(Configuration conf) { + return conf.getBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, + DataTieringManager.DEFAULT_GLOBAL_DATA_TIERING_ENABLED); + } + + // Resets the instance to null. To be used only for testing. + public static void resetForTestingOnly() { + instance = null; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index f1615e5e1e91..8fa555cc088d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -700,7 +700,9 @@ public HRegionServer(final Configuration conf) throws IOException { // no need to instantiate block cache and mob file cache when master not carry table if (!isMasterNotCarryTable) { blockCache = BlockCacheFactory.createBlockCache(conf); - DataTieringManager.instantiate(onlineRegions); + // The call below, instantiates the DataTieringManager only when + // the configuration "hbase.regionserver.datatiering.enable" is set to true. + DataTieringManager.instantiate(conf,onlineRegions); mobFileCache = new MobFileCache(conf); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 9c8073961b8b..b74937bf94de 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; import static org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.DEFAULT_ERROR_TOLERATION_DURATION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -66,6 +68,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is used to test the functionality of the DataTieringManager. @@ -94,6 +98,7 @@ public class TestDataTieringManager { HBaseClassTestRule.forClass(TestDataTieringManager.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Logger LOG = LoggerFactory.getLogger(TestDataTieringManager.class); private static Configuration defaultConf; private static FileSystem fs; private static BlockCache blockCache; @@ -118,10 +123,11 @@ public static void setupBeforeClass() throws Exception { defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32); + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); fs = HFileSystem.get(defaultConf); blockCache = BlockCacheFactory.createBlockCache(defaultConf); cacheConf = new CacheConfig(defaultConf, blockCache); - DataTieringManager.instantiate(testOnlineRegions); + assertTrue(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); setupOnlineRegions(); dataTieringManager = DataTieringManager.getInstance(); } @@ -283,9 +289,9 @@ public void testBlockEvictions() throws Exception { int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; // Setup: Create a bucket cache with lower capacity - BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, - 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", - DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); // Create three Cache keys with cold data files and a block with hot data. // hStoreFiles.get(3) is a cold data file, while hStoreFiles.get(0) is a hot file. @@ -333,9 +339,9 @@ public void testBlockEvictionsAllColdBlocks() throws Exception { int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; // Setup: Create a bucket cache with lower capacity - BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, - 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", - DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); // Create three Cache keys with three cold data blocks. // hStoreFiles.get(3) is a cold data file. @@ -380,9 +386,9 @@ public void testBlockEvictionsHotBlocks() throws Exception { int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; // Setup: Create a bucket cache with lower capacity - BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, - 8192, bucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence", - DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); // Create three Cache keys with two hot data blocks and one cold data block // hStoreFiles.get(0) is a hot data file and hStoreFiles.get(3) is a cold data file. @@ -417,11 +423,74 @@ public void testBlockEvictionsHotBlocks() throws Exception { validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0); } + @Test + public void testFeatureKeyDisabled() throws Exception { + DataTieringManager.resetForTestingOnly(); + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, false); + try { + assertFalse(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); + // Verify that the DataaTieringManager instance is not instantiated in the + // instantiate call above. + assertNull(DataTieringManager.getInstance()); + + // Also validate that data temperature is not honoured. + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with two hot data blocks and one cold data block + // hStoreFiles.get(0) is a hot data file and hStoreFiles.get(3) is a cold data file. + List cacheKeys = new ArrayList<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + LOG.info("Adding {}", key); + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional hot block, which triggers eviction. + BlockCacheKey newKey = + new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket still contains the only cold block and one newly added hot block. + // The older hot blocks are evicted and data-tiering mechanism does not kick in to evict + // the cold block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 1, 1); + } finally { + DataTieringManager.resetForTestingOnly(); + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); + assertTrue(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); + } + } + private void validateBlocks(Set keys, int expectedTotalKeys, int expectedHotBlocks, int expectedColdBlocks) { int numHotBlocks = 0, numColdBlocks = 0; - assertEquals(expectedTotalKeys, keys.size()); + Waiter.waitFor(defaultConf, 10000, 100, () -> (expectedTotalKeys == keys.size())); int iter = 0; for (BlockCacheKey key : keys) { try { From 56dc8bb6baf187d02c8c672310cf3ed3786fad0f Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Wed, 22 May 2024 18:59:49 +0530 Subject: [PATCH 045/336] HBASE-28469: Integration of time-based priority caching into compaction paths (#5866) Signed-off-by: Wellington Chevreuil Reviewed-by: Janardhan Hugund Change-Id: Ib992689f769774a2af5fc3f98af892e926b0f7bf --- .../hadoop/hbase/io/hfile/BlockCache.java | 17 +++ .../hadoop/hbase/io/hfile/BlockCacheKey.java | 1 - .../hbase/io/hfile/CombinedBlockCache.java | 20 ++- .../apache/hadoop/hbase/io/hfile/HFile.java | 5 + .../hbase/io/hfile/HFileWriterImpl.java | 50 +++++++ .../hbase/io/hfile/bucket/BucketCache.java | 13 ++ .../regionserver/DataTieringManager.java | 50 ++++++- .../hbase/regionserver/HRegionFileSystem.java | 25 ++++ .../hbase/regionserver/StoreFileWriter.java | 23 +--- .../hbase/regionserver/TimeRangeTracker.java | 4 +- .../regionserver/TestDataTieringManager.java | 130 ++++++++++++++---- 11 files changed, 281 insertions(+), 57 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 8380fc194e7f..90fb7ce34916 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.conf.ConfigurationObserver; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; @@ -193,6 +194,22 @@ default Optional shouldCacheFile(HFileInfo hFileInfo, Configuration con return Optional.empty(); } + /** + * Checks whether the block represented by the given key should be cached or not. This method may + * not be overridden by all implementing classes. In such cases, the returned Optional will be + * empty. For subclasses implementing this logic, the returned Optional would contain the boolean + * value reflecting if the passed block should indeed be cached. + * @param key The key representing the block to check if it should be cached. + * @param timeRangeTracker the time range tracker containing the timestamps + * @param conf The configuration object to use for determining caching behavior. + * @return An empty Optional if this method is not supported; otherwise, the returned Optional + * contains the boolean value indicating if the block should be cached. + */ + default Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + Configuration conf) { + return Optional.empty(); + } + /** * Checks whether the block for the passed key is already cached. This method may not be * overridden by all implementing classes. In such cases, the returned Optional will be empty. For diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java index bf22d38e373b..bcc1f58ba5e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java @@ -116,5 +116,4 @@ public void setBlockType(BlockType blockType) { public Path getFilePath() { return filePath; } - } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index fe675aade7bb..cb3000dda196 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -478,11 +479,22 @@ public Optional blockFitsIntoTheCache(HFileBlock block) { @Override public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf) { - Optional l1Result = l1Cache.shouldCacheFile(hFileInfo, conf); - Optional l2Result = l2Cache.shouldCacheFile(hFileInfo, conf); + return combineCacheResults(l1Cache.shouldCacheFile(hFileInfo, conf), + l2Cache.shouldCacheFile(hFileInfo, conf)); + } + + @Override + public Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + Configuration conf) { + return combineCacheResults(l1Cache.shouldCacheBlock(key, timeRangeTracker, conf), + l2Cache.shouldCacheBlock(key, timeRangeTracker, conf)); + } + + private Optional combineCacheResults(Optional result1, + Optional result2) { final Mutable combinedResult = new MutableBoolean(true); - l1Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); - l2Result.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); + result1.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); + result2.ifPresent(b -> combinedResult.setValue(b && combinedResult.getValue())); return Optional.of(combinedResult.getValue()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index ae79ad857244..2c3908aa33f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -212,6 +212,11 @@ public interface Writer extends Closeable, CellSink, ShipperListener { /** Add an element to the file info map. */ void appendFileInfo(byte[] key, byte[] value) throws IOException; + /** + * Add TimestampRange and earliest put timestamp to Metadata + */ + void appendTrackedTimestampsToMetadata() throws IOException; + /** Returns the path to this {@link HFile} */ Path getPath(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java index d2dfaf62106a..44ec324686e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.io.hfile; import static org.apache.hadoop.hbase.io.hfile.BlockCompressedSizePredicator.MAX_BLOCK_SIZE_UNCOMPRESSED; +import static org.apache.hadoop.hbase.regionserver.HStoreFile.EARLIEST_PUT_TS; +import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; import java.io.DataOutput; import java.io.DataOutputStream; @@ -26,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -45,6 +48,7 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.security.EncryptionUtil; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.BloomFilterWriter; @@ -117,6 +121,8 @@ public class HFileWriterImpl implements HFile.Writer { /** May be null if we were passed a stream. */ protected final Path path; + protected final Configuration conf; + /** Cache configuration for caching data on write. */ protected final CacheConfig cacheConf; @@ -170,12 +176,16 @@ public class HFileWriterImpl implements HFile.Writer { protected long maxMemstoreTS = 0; + private final TimeRangeTracker timeRangeTracker; + private long earliestPutTs = HConstants.LATEST_TIMESTAMP; + public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path, FSDataOutputStream outputStream, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; + this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); @@ -190,6 +200,7 @@ public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path pat } closeOutputStream = path != null; this.cacheConf = cacheConf; + this.conf = conf; float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 0f); this.encodedBlockSizeLimit = (int) (hFileContext.getBlocksize() * encodeBlockSizeRatio); @@ -555,6 +566,10 @@ private void writeInlineBlocks(boolean closing) throws IOException { private void doCacheOnWrite(long offset) { cacheConf.getBlockCache().ifPresent(cache -> { HFileBlock cacheFormatBlock = blockWriter.getBlockForCaching(cacheConf); + BlockCacheKey key = buildCacheBlockKey(offset, cacheFormatBlock.getBlockType()); + if (!shouldCacheBlock(cache, key)) { + return; + } try { cache.cacheBlock(new BlockCacheKey(name, offset, true, cacheFormatBlock.getBlockType()), cacheFormatBlock, cacheConf.isInMemory(), true); @@ -565,6 +580,18 @@ private void doCacheOnWrite(long offset) { }); } + private BlockCacheKey buildCacheBlockKey(long offset, BlockType blockType) { + if (path != null) { + return new BlockCacheKey(path, offset, true, blockType); + } + return new BlockCacheKey(name, offset, true, blockType); + } + + private boolean shouldCacheBlock(BlockCache cache, BlockCacheKey key) { + Optional result = cache.shouldCacheBlock(key, timeRangeTracker, conf); + return result.orElse(true); + } + /** * Ready a new block for writing. */ @@ -767,6 +794,8 @@ public void append(final Cell cell) throws IOException { if (tagsLength > this.maxTagsLength) { this.maxTagsLength = tagsLength; } + + trackTimestamps(cell); } @Override @@ -859,4 +888,25 @@ protected void finishClose(FixedFileTrailer trailer) throws IOException { outputStream = null; } } + + /** + * Add TimestampRange and earliest put timestamp to Metadata + */ + public void appendTrackedTimestampsToMetadata() throws IOException { + // TODO: The StoreFileReader always converts the byte[] to TimeRange + // via TimeRangeTracker, so we should write the serialization data of TimeRange directly. + appendFileInfo(TIMERANGE_KEY, TimeRangeTracker.toByteArray(timeRangeTracker)); + appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); + } + + /** + * Record the earliest Put timestamp. If the timeRangeTracker is not set, update TimeRangeTracker + * to include the timestamp of this key + */ + private void trackTimestamps(final Cell cell) { + if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) { + earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp()); + } + timeRangeTracker.includeTimestamp(cell); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 7242e40bf05a..f39032840678 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.regionserver.DataTieringManager; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.IdReadWriteLock; @@ -2363,6 +2364,18 @@ public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf return Optional.of(!fullyCachedFiles.containsKey(fileName)); } + @Override + public Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + Configuration conf) { + DataTieringManager dataTieringManager = DataTieringManager.getInstance(); + if (dataTieringManager != null && !dataTieringManager.isHotData(timeRangeTracker, conf)) { + LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", + key.getHfileName()); + return Optional.of(false); + } + return Optional.of(true); + } + @Override public Optional isAlreadyCached(BlockCacheKey key) { boolean foundKey = backingMap.containsKey(key); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index d3bdbf330cc5..f71bc5e43aa6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -131,6 +131,27 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException { return isHotData(hFilePath); } + /** + * Determines whether the data associated with the given time range tracker is considered hot. If + * the data tiering type is set to {@link DataTieringType#TIME_RANGE}, it uses the maximum + * timestamp from the time range tracker to determine if the data is hot. Otherwise, it considers + * the data as hot by default. + * @param timeRangeTracker the time range tracker containing the timestamps + * @param conf The configuration object to use for determining hot data criteria. + * @return {@code true} if the data is hot, {@code false} otherwise + */ + public boolean isHotData(TimeRangeTracker timeRangeTracker, Configuration conf) { + DataTieringType dataTieringType = getDataTieringType(conf); + if ( + dataTieringType.equals(DataTieringType.TIME_RANGE) + && timeRangeTracker.getMax() != TimeRangeTracker.INITIAL_MAX_TIMESTAMP + ) { + return hotDataValidator(timeRangeTracker.getMax(), getDataTieringHotDataAge(conf)); + } + // DataTieringType.NONE or other types are considered hot by default + return true; + } + /** * Determines whether the data in the HFile at the given path is considered hot based on the * configured data tiering type and hot data age. If the data tiering type is set to @@ -151,6 +172,27 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { return true; } + /** + * Determines whether the data in the HFile at the given path is considered hot based on the + * configured data tiering type and hot data age. If the data tiering type is set to + * {@link DataTieringType#TIME_RANGE}, it validates the data against the provided maximum + * timestamp. + * @param hFilePath the path to the HFile + * @param maxTimestamp the maximum timestamp to validate against + * @return {@code true} if the data is hot, {@code false} otherwise + * @throws DataTieringException if there is an error retrieving data tiering information + */ + public boolean isHotData(Path hFilePath, long maxTimestamp) throws DataTieringException { + Configuration configuration = getConfiguration(hFilePath); + DataTieringType dataTieringType = getDataTieringType(configuration); + + if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); + } + // DataTieringType.NONE or other types are considered hot by default + return true; + } + /** * Determines whether the data in the HFile being read is considered hot based on the configured * data tiering type and hot data age. If the data tiering type is set to @@ -231,10 +273,12 @@ public Set getColdDataFiles(Set allCachedBlocks) } private HRegion getHRegion(Path hFilePath) throws DataTieringException { - if (hFilePath.getParent() == null || hFilePath.getParent().getParent() == null) { - throw new DataTieringException("Incorrect HFile Path: " + hFilePath); + String regionId; + try { + regionId = HRegionFileSystem.getRegionId(hFilePath); + } catch (IOException e) { + throw new DataTieringException(e.getMessage()); } - String regionId = hFilePath.getParent().getParent().getName(); HRegion hRegion = this.onlineRegions.get(regionId); if (hRegion == null) { throw new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index f7144c7fa9dd..c77f4d4aefde 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -1067,6 +1067,31 @@ public static void deleteRegionFromFileSystem(final Configuration conf, final Fi } } + /** + * Retrieves the Region ID from the given HFile path. + * @param hFilePath The path of the HFile. + * @return The Region ID extracted from the HFile path. + * @throws IOException If an I/O error occurs or if the HFile path is incorrect. + */ + public static String getRegionId(Path hFilePath) throws IOException { + if (hFilePath.getParent() == null || hFilePath.getParent().getParent() == null) { + throw new IOException("Incorrect HFile Path: " + hFilePath); + } + Path dir = hFilePath.getParent().getParent(); + if (isTemporaryDirectoryName(dir.getName())) { + if (dir.getParent() == null) { + throw new IOException("Incorrect HFile Path: " + hFilePath); + } + return dir.getParent().getName(); + } + return dir.getName(); + } + + private static boolean isTemporaryDirectoryName(String dirName) { + return REGION_MERGES_DIR.equals(dirName) || REGION_SPLITS_DIR.equals(dirName) + || REGION_TEMP_DIR.equals(dirName); + } + /** * Creates a directory. Assumes the user has already checked for this directory existence. * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java index a2a641df6d8f..5f5fcf2001a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java @@ -22,13 +22,11 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BLOOM_FILTER_TYPE_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.COMPACTION_EVENT_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.DELETE_FAMILY_COUNT; -import static org.apache.hadoop.hbase.regionserver.HStoreFile.EARLIEST_PUT_TS; import static org.apache.hadoop.hbase.regionserver.HStoreFile.HISTORICAL_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_FILE_REFS; -import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; import static org.apache.hadoop.hbase.regionserver.StoreEngine.STORE_ENGINE_CLASS_KEY; import java.io.IOException; @@ -52,7 +50,6 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; @@ -497,11 +494,9 @@ private static final class SingleStoreFileWriter { private final BloomFilterWriter deleteFamilyBloomFilterWriter; private final BloomType bloomType; private byte[] bloomParam = null; - private long earliestPutTs = HConstants.LATEST_TIMESTAMP; private long deleteFamilyCnt = 0; private BloomContext bloomContext = null; private BloomContext deleteFamilyBloomContext = null; - private final TimeRangeTracker timeRangeTracker; private final Supplier> compactedFilesSupplier; private HFile.Writer writer; @@ -525,7 +520,6 @@ private SingleStoreFileWriter(FileSystem fs, Path path, final Configuration conf HFileContext fileContext, boolean shouldDropCacheBehind, Supplier> compactedFilesSupplier) throws IOException { this.compactedFilesSupplier = compactedFilesSupplier; - this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); // TODO : Change all writers to be specifically created for compaction context writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, path).withFavoredNodes(favoredNodes) @@ -667,21 +661,7 @@ private void appendMobMetadata(SetMultimap mobRefSet) throws * Add TimestampRange and earliest put timestamp to Metadata */ private void appendTrackedTimestampsToMetadata() throws IOException { - // TODO: The StoreFileReader always converts the byte[] to TimeRange - // via TimeRangeTracker, so we should write the serialization data of TimeRange directly. - appendFileInfo(TIMERANGE_KEY, TimeRangeTracker.toByteArray(timeRangeTracker)); - appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); - } - - /** - * Record the earlest Put timestamp. If the timeRangeTracker is not set, update TimeRangeTracker - * to include the timestamp of this key - */ - private void trackTimestamps(final Cell cell) { - if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) { - earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp()); - } - timeRangeTracker.includeTimestamp(cell); + writer.appendTrackedTimestampsToMetadata(); } private void appendGeneralBloomfilter(final Cell cell) throws IOException { @@ -712,7 +692,6 @@ private void append(final Cell cell) throws IOException { appendGeneralBloomfilter(cell); appendDeleteFamilyBloomFilter(cell); writer.append(cell); - trackTimestamps(cell); } private void beforeShipped() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java index 7fc79642d919..53deb7e9cea7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java @@ -53,8 +53,8 @@ public enum Type { SYNC } - static final long INITIAL_MIN_TIMESTAMP = Long.MAX_VALUE; - static final long INITIAL_MAX_TIMESTAMP = -1L; + public static final long INITIAL_MIN_TIMESTAMP = Long.MAX_VALUE; + public static final long INITIAL_MAX_TIMESTAMP = -1L; public static TimeRangeTracker create(Type type) { switch (type) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index b74937bf94de..315d88d3836d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -30,8 +30,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Optional; +import java.util.Random; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -57,8 +57,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; -import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -99,6 +97,7 @@ public class TestDataTieringManager { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final Logger LOG = LoggerFactory.getLogger(TestDataTieringManager.class); + private static final long DAY = 24 * 60 * 60 * 1000; private static Configuration defaultConf; private static FileSystem fs; private static BlockCache blockCache; @@ -120,16 +119,16 @@ public class TestDataTieringManager { public static void setupBeforeClass() throws Exception { testDir = TEST_UTIL.getDataTestDir(TestDataTieringManager.class.getSimpleName()); defaultConf = TEST_UTIL.getConfiguration(); - defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); - defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); - defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32); - defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); - fs = HFileSystem.get(defaultConf); - blockCache = BlockCacheFactory.createBlockCache(defaultConf); - cacheConf = new CacheConfig(defaultConf, blockCache); + updateCommonConfigurations(); assertTrue(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); - setupOnlineRegions(); dataTieringManager = DataTieringManager.getInstance(); + rowKeyString = ""; + } + + private static void updateCommonConfigurations() { + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); + defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32); } @FunctionalInterface @@ -143,7 +142,8 @@ interface DataTieringMethodCallerWithKey { } @Test - public void testDataTieringEnabledWithKey() { + public void testDataTieringEnabledWithKey() throws IOException { + initializeTestEnvironment(); DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isDataTieringEnabled; // Test with valid key @@ -161,7 +161,8 @@ public void testDataTieringEnabledWithKey() { } @Test - public void testDataTieringEnabledWithPath() { + public void testDataTieringEnabledWithPath() throws IOException { + initializeTestEnvironment(); DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isDataTieringEnabled; // Test with valid path @@ -191,7 +192,8 @@ public void testDataTieringEnabledWithPath() { } @Test - public void testHotDataWithKey() { + public void testHotDataWithKey() throws IOException { + initializeTestEnvironment(); DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isHotData; // Test with valid key @@ -204,7 +206,8 @@ public void testHotDataWithKey() { } @Test - public void testHotDataWithPath() { + public void testHotDataWithPath() throws IOException { + initializeTestEnvironment(); DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData; // Test with valid path @@ -244,7 +247,8 @@ private void setPrefetchBlocksOnOpen() { } @Test - public void testColdDataFiles() { + public void testColdDataFiles() throws IOException { + initializeTestEnvironment(); Set allCachedBlocks = new HashSet<>(); for (HStoreFile file : hStoreFiles) { allCachedBlocks.add(new BlockCacheKey(file.getPath(), 0, true, BlockType.DATA)); @@ -270,7 +274,74 @@ public void testColdDataFiles() { } @Test - public void testPickColdDataFiles() { + public void testCacheCompactedBlocksOnWriteDataTieringDisabled() throws IOException { + setCacheCompactBlocksOnWrite(); + initializeTestEnvironment(); + + HRegion region = createHRegion("table3"); + testCacheCompactedBlocksOnWrite(region, true); + } + + @Test + public void testCacheCompactedBlocksOnWriteWithHotData() throws IOException { + setCacheCompactBlocksOnWrite(); + initializeTestEnvironment(); + + HRegion region = createHRegion("table3", getConfWithTimeRangeDataTieringEnabled(5 * DAY)); + testCacheCompactedBlocksOnWrite(region, true); + } + + @Test + public void testCacheCompactedBlocksOnWriteWithColdData() throws IOException { + setCacheCompactBlocksOnWrite(); + initializeTestEnvironment(); + + HRegion region = createHRegion("table3", getConfWithTimeRangeDataTieringEnabled(DAY)); + testCacheCompactedBlocksOnWrite(region, false); + } + + private void setCacheCompactBlocksOnWrite() { + defaultConf.setBoolean(CacheConfig.CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, true); + } + + private void testCacheCompactedBlocksOnWrite(HRegion region, boolean expectDataBlocksCached) + throws IOException { + HStore hStore = createHStore(region, "cf1"); + createTestFilesForCompaction(hStore); + hStore.refreshStoreFiles(); + + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + + long initialStoreFilesCount = hStore.getStorefilesCount(); + long initialCacheDataBlockCount = blockCache.getDataBlockCount(); + assertEquals(3, initialStoreFilesCount); + assertEquals(0, initialCacheDataBlockCount); + + region.compact(true); + + long compactedStoreFilesCount = hStore.getStorefilesCount(); + long compactedCacheDataBlockCount = blockCache.getDataBlockCount(); + assertEquals(1, compactedStoreFilesCount); + assertEquals(expectDataBlocksCached, compactedCacheDataBlockCount > 0); + } + + private void createTestFilesForCompaction(HStore hStore) throws IOException { + long currentTime = System.currentTimeMillis(); + Path storeDir = hStore.getStoreContext().getFamilyStoreDirectoryPath(); + Configuration configuration = hStore.getReadOnlyConfiguration(); + + createHStoreFile(storeDir, configuration, currentTime - 2 * DAY, + hStore.getHRegion().getRegionFileSystem()); + createHStoreFile(storeDir, configuration, currentTime - 3 * DAY, + hStore.getHRegion().getRegionFileSystem()); + createHStoreFile(storeDir, configuration, currentTime - 4 * DAY, + hStore.getHRegion().getRegionFileSystem()); + } + + @Test + public void testPickColdDataFiles() throws IOException { + initializeTestEnvironment(); Map coldDataFiles = dataTieringManager.getColdFilesList(); assertEquals(1, coldDataFiles.size()); // hStoreFiles[3] is the cold file. @@ -283,6 +354,7 @@ public void testPickColdDataFiles() { */ @Test public void testBlockEvictions() throws Exception { + initializeTestEnvironment(); long capacitySize = 40 * 1024; int writeThreads = 3; int writerQLen = 64; @@ -333,6 +405,7 @@ public void testBlockEvictions() throws Exception { */ @Test public void testBlockEvictionsAllColdBlocks() throws Exception { + initializeTestEnvironment(); long capacitySize = 40 * 1024; int writeThreads = 3; int writerQLen = 64; @@ -380,6 +453,7 @@ public void testBlockEvictionsAllColdBlocks() throws Exception { */ @Test public void testBlockEvictionsHotBlocks() throws Exception { + initializeTestEnvironment(); long capacitySize = 40 * 1024; int writeThreads = 3; int writerQLen = 64; @@ -427,6 +501,8 @@ public void testBlockEvictionsHotBlocks() throws Exception { public void testFeatureKeyDisabled() throws Exception { DataTieringManager.resetForTestingOnly(); defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, false); + initializeTestEnvironment(); + try { assertFalse(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); // Verify that the DataaTieringManager instance is not instantiated in the @@ -632,7 +708,12 @@ private static HRegion createHRegion(String table, Configuration conf) throws IO HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); - return new HRegion(regionFs, null, conf, htd, null); + HRegion region = new HRegion(regionFs, null, conf, htd, null); + // Manually sets the BlockCache for the HRegion instance. + // This is necessary because the region server is not started within this method, + // and therefore the BlockCache needs to be explicitly configured. + region.setBlockCache(blockCache); + return region; } private static HStore createHStore(HRegion region, String columnFamily) throws IOException { @@ -668,13 +749,14 @@ static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long times writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); - StoreContext storeContext = - StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); - - StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); - return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, sft); + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true); } + /** + * Writes random data to a store file with rows arranged in lexicographically increasing order. + * Each row is generated using the {@link #nextString()} method, ensuring that each subsequent row + * is lexicographically larger than the previous one. + */ private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily, long timestamp) throws IOException { int cellsPerFile = 10; @@ -691,7 +773,6 @@ private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[ } } - private static byte[] generateRandomBytes(int sizeInBytes) { Random random = new Random(); byte[] randomBytes = new byte[sizeInBytes]; @@ -714,5 +795,4 @@ private static String nextString() { } return rowKeyString; } - } From 502f0baa7fcfce5e5a147d9da27a48923a87e58f Mon Sep 17 00:00:00 2001 From: jhungund <106576553+jhungund@users.noreply.github.com> Date: Thu, 23 May 2024 00:16:49 +0530 Subject: [PATCH 046/336] HBASE-28467: Add time-based priority caching checks for cacheOnRead code paths. (#5905) Signed-off-by: Wellington Chevreuil --- .../hadoop/hbase/io/hfile/CacheConfig.java | 12 ++++ .../hbase/io/hfile/HFileReaderImpl.java | 6 +- .../regionserver/TestDataTieringManager.java | 64 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 94ee48dd76b4..4f57668a0c6d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -266,6 +266,18 @@ public boolean shouldCacheBlockOnRead(BlockCategory category) { || (prefetchOnOpen && (category != BlockCategory.META && category != BlockCategory.UNKNOWN)); } + public boolean shouldCacheBlockOnRead(BlockCategory category, HFileInfo hFileInfo, + Configuration conf) { + Optional cacheFileBlock = Optional.of(true); + if (getBlockCache().isPresent()) { + Optional result = getBlockCache().get().shouldCacheFile(hFileInfo, conf); + if (result.isPresent()) { + cacheFileBlock = result; + } + } + return shouldCacheBlockOnRead(category) && cacheFileBlock.get(); + } + /** Returns true if blocks in this file should be flagged as in-memory */ public boolean isInMemory() { return this.inMemory; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index e1e9eaf8a53a..6aa16f9423cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -1231,7 +1231,8 @@ public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock) throws BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, this.isPrimaryReplicaReader(), BlockType.META); - cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory()); + cacheBlock &= + cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory(), getHFileInfo(), conf); HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, BlockType.META, null); if (cachedBlock != null) { @@ -1386,7 +1387,8 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo } BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory(); final boolean cacheCompressed = cacheConf.shouldCacheCompressed(category); - final boolean cacheOnRead = cacheConf.shouldCacheBlockOnRead(category); + final boolean cacheOnRead = + cacheConf.shouldCacheBlockOnRead(category, getHFileInfo(), conf); // Don't need the unpacked block back and we're storing the block in the cache compressed if (cacheOnly && cacheCompressed && cacheOnRead) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 315d88d3836d..b24c1f2ed84a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.DEFAULT_ERROR_TOLERATION_DURATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -49,12 +50,15 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -562,6 +566,66 @@ public void testFeatureKeyDisabled() throws Exception { } } + @Test + public void testCacheConfigShouldCacheFile() throws Exception { + // Evict the files from cache. + for (HStoreFile file : hStoreFiles) { + file.closeStoreFile(true); + } + // Verify that the API shouldCacheFileBlock returns the result correctly. + // hStoreFiles[0], hStoreFiles[1], hStoreFiles[2] are hot files. + // hStoreFiles[3] is a cold file. + try { + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(0).getFileInfo().getHFileInfo(), + hStoreFiles.get(0).getFileInfo().getConf())); + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(1).getFileInfo().getHFileInfo(), + hStoreFiles.get(1).getFileInfo().getConf())); + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(2).getFileInfo().getHFileInfo(), + hStoreFiles.get(2).getFileInfo().getConf())); + assertFalse(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(3).getFileInfo().getHFileInfo(), + hStoreFiles.get(3).getFileInfo().getConf())); + } finally { + for (HStoreFile file : hStoreFiles) { + file.initReader(); + } + } + } + + @Test + public void testCacheOnReadColdFile() throws Exception { + // hStoreFiles[3] is a cold file. the blocks should not get loaded after a readBlock call. + HStoreFile hStoreFile = hStoreFiles.get(3); + BlockCacheKey cacheKey = new BlockCacheKey(hStoreFile.getPath(), 0, true, BlockType.DATA); + testCacheOnRead(hStoreFile, cacheKey, 23025, false); + } + + @Test + public void testCacheOnReadHotFile() throws Exception { + // hStoreFiles[0] is a hot file. the blocks should get loaded after a readBlock call. + HStoreFile hStoreFile = hStoreFiles.get(0); + BlockCacheKey cacheKey = + new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testCacheOnRead(hStoreFile, cacheKey, 23025, true); + } + + private void testCacheOnRead(HStoreFile hStoreFile, BlockCacheKey key, long onDiskBlockSize, + boolean expectedCached) throws Exception { + // Execute the read block API which will try to cache the block if the block is a hot block. + hStoreFile.getReader().getHFileReader().readBlock(key.getOffset(), onDiskBlockSize, true, false, + false, false, key.getBlockType(), DataBlockEncoding.NONE); + // Validate that the hot block gets cached and cold block is not cached. + HFileBlock block = (HFileBlock) blockCache.getBlock(key, false, false, false, BlockType.DATA); + if (expectedCached) { + assertNotNull(block); + } else { + assertNull(block); + } + } + private void validateBlocks(Set keys, int expectedTotalKeys, int expectedHotBlocks, int expectedColdBlocks) { int numHotBlocks = 0, numColdBlocks = 0; From b704348dac2269cc630ac854e7449a386273bb75 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 15 Jul 2025 11:48:35 +0100 Subject: [PATCH 047/336] HBASE-29427 Merge all commits related to custom tiering into the feature branch (#7124) This is the whole custom tiering implementation and involves the following individual works: * HBASE-29412 Extend date tiered compaction to allow for tiering by values other than cell timestamp * HBASE-29413 Implement a custom qualifier tiered compaction * HBASE-29414 Refactor DataTieringManager to make priority logic pluggable * HBASE-29422 Implement selectMinorCompation in CustomCellDateTieredCompactionPolicy * HBASE-29424 Implement configuration validation for custom tiering compactions * HBASE-29425 Refine and polish code * HBASE-29426 Define a tiering value provider and refactor custom tiered compaction related classes * HBASE-28463 Rebase time based priority branch (HBASE-28463) with latest master (and fix conflicts) Co-authored-by: Janardhan Hungund Signed-off-by: Tak Lon (Stephen) Wu Change-Id: I5dc170e31e47b297e1d81f07d625f680c5e0d848 --- .../java/org/apache/hadoop/hbase/TagType.java | 2 + .../hadoop/hbase/io/hfile/BlockCache.java | 9 +- .../hadoop/hbase/io/hfile/CacheConfig.java | 3 +- .../hbase/io/hfile/CombinedBlockCache.java | 7 +- .../apache/hadoop/hbase/io/hfile/HFile.java | 7 + .../hbase/io/hfile/HFileReaderImpl.java | 4 +- .../hbase/io/hfile/HFileWriterImpl.java | 20 +- .../hbase/io/hfile/bucket/BucketCache.java | 10 +- .../procedure/CreateTableProcedure.java | 3 + .../procedure/ModifyTableProcedure.java | 2 + .../hbase/regionserver/CellTSTiering.java | 57 ++ .../regionserver/CustomTieredStoreEngine.java | 56 ++ .../hbase/regionserver/CustomTiering.java | 58 ++ .../CustomTieringMultiFileWriter.java | 85 ++ .../hbase/regionserver/DataTiering.java | 29 + .../regionserver/DataTieringManager.java | 98 +- .../hbase/regionserver/DataTieringType.java | 15 +- .../DateTieredMultiFileWriter.java | 20 +- .../regionserver/DateTieredStoreEngine.java | 17 +- .../hbase/regionserver/HRegionServer.java | 4 +- .../hbase/regionserver/StoreFileWriter.java | 13 + .../regionserver/compactions/Compactor.java | 6 + .../compactions/CustomCellTieredUtils.java | 49 + .../CustomCellTieringValueProvider.java | 86 ++ .../CustomDateTieredCompactionPolicy.java | 155 ++++ .../compactions/CustomTieredCompactor.java | 74 ++ .../DateTieredCompactionPolicy.java | 129 ++- .../compactions/DateTieredCompactor.java | 12 +- .../client/TestIllegalTableDescriptor.java | 14 +- .../hbase/io/hfile/TestBytesReadFromFs.java | 4 + .../TestCustomCellDataTieringManager.java | 860 ++++++++++++++++++ .../TestCustomCellTieredCompactionPolicy.java | 267 ++++++ .../regionserver/TestDataTieringManager.java | 13 +- .../TestCustomCellTieredCompactor.java | 148 +++ 34 files changed, 2175 insertions(+), 161 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellTSTiering.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieredStoreEngine.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTiering.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieringMultiFileWriter.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTiering.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieredUtils.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieringValueProvider.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomDateTieredCompactionPolicy.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomTieredCompactor.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCustomCellTieredCompactor.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index eb9a7f3eccc9..b0df4920e4ed 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -36,4 +36,6 @@ public final class TagType { // String based tag type used in replication public static final byte STRING_VIS_TAG_TYPE = (byte) 7; public static final byte TTL_TAG_TYPE = (byte) 8; + // tag with the custom cell tiering value for the row + public static final byte CELL_VALUE_TIERING_TAG_TYPE = (byte) 9; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 90fb7ce34916..313b4034fb86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.conf.ConfigurationObserver; -import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; @@ -199,13 +198,13 @@ default Optional shouldCacheFile(HFileInfo hFileInfo, Configuration con * not be overridden by all implementing classes. In such cases, the returned Optional will be * empty. For subclasses implementing this logic, the returned Optional would contain the boolean * value reflecting if the passed block should indeed be cached. - * @param key The key representing the block to check if it should be cached. - * @param timeRangeTracker the time range tracker containing the timestamps - * @param conf The configuration object to use for determining caching behavior. + * @param key The key representing the block to check if it should be cached. + * @param maxTimeStamp The maximum timestamp for the block to check if it should be cached. + * @param conf The configuration object to use for determining caching behavior. * @return An empty Optional if this method is not supported; otherwise, the returned Optional * contains the boolean value indicating if the block should be cached. */ - default Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + default Optional shouldCacheBlock(BlockCacheKey key, long maxTimeStamp, Configuration conf) { return Optional.empty(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 4f57668a0c6d..b6357ca94b86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -269,7 +269,8 @@ public boolean shouldCacheBlockOnRead(BlockCategory category) { public boolean shouldCacheBlockOnRead(BlockCategory category, HFileInfo hFileInfo, Configuration conf) { Optional cacheFileBlock = Optional.of(true); - if (getBlockCache().isPresent()) { + // For DATA blocks only, if BuckeCache is in use, we don't need to cache block again + if (getBlockCache().isPresent() && category.equals(BlockCategory.DATA)) { Optional result = getBlockCache().get().shouldCacheFile(hFileInfo, conf); if (result.isPresent()) { cacheFileBlock = result; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index cb3000dda196..856b5da6d117 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -484,10 +483,10 @@ public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf } @Override - public Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + public Optional shouldCacheBlock(BlockCacheKey key, long maxTimeStamp, Configuration conf) { - return combineCacheResults(l1Cache.shouldCacheBlock(key, timeRangeTracker, conf), - l2Cache.shouldCacheBlock(key, timeRangeTracker, conf)); + return combineCacheResults(l1Cache.shouldCacheBlock(key, maxTimeStamp, conf), + l2Cache.shouldCacheBlock(key, maxTimeStamp, conf)); } private Optional combineCacheResults(Optional result1, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 2c3908aa33f3..1f8f84215488 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.CellSink; import org.apache.hadoop.hbase.regionserver.ShipperListener; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -217,6 +218,12 @@ public interface Writer extends Closeable, CellSink, ShipperListener { */ void appendTrackedTimestampsToMetadata() throws IOException; + /** + * Add Custom cell timestamp to Metadata + */ + public void appendCustomCellTimestampsToMetadata(TimeRangeTracker timeRangeTracker) + throws IOException; + /** Returns the path to this {@link HFile} */ Path getPath(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 6aa16f9423cd..e837e8f1bd6b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -1396,7 +1396,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo cacheConf.getBlockCache().ifPresent(cache -> { LOG.debug("Skipping decompression of block {} in prefetch", cacheKey); // Cache the block if necessary - if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { + if (cacheBlock && cacheOnRead) { cache.cacheBlock(cacheKey, blockNoChecksum, cacheConf.isInMemory(), cacheOnly); } }); @@ -1410,7 +1410,7 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final bo HFileBlock unpackedNoChecksum = BlockCacheUtil.getBlockForCaching(cacheConf, unpacked); // Cache the block if necessary cacheConf.getBlockCache().ifPresent(cache -> { - if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { + if (cacheBlock && cacheOnRead) { // Using the wait on cache during compaction and prefetching. cache.cacheBlock(cacheKey, cacheCompressed diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java index 44ec324686e8..2bdfcfa45b34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.io.hfile; import static org.apache.hadoop.hbase.io.hfile.BlockCompressedSizePredicator.MAX_BLOCK_SIZE_UNCOMPRESSED; +import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EARLIEST_PUT_TS; import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -126,6 +128,12 @@ public class HFileWriterImpl implements HFile.Writer { /** Cache configuration for caching data on write. */ protected final CacheConfig cacheConf; + public void setTimeRangeTrackerForTiering(Supplier timeRangeTrackerForTiering) { + this.timeRangeTrackerForTiering = timeRangeTrackerForTiering; + } + + private Supplier timeRangeTrackerForTiering; + /** * Name for this object used when logging or in toString. Is either the result of a toString on * stream or else name of passed file Path. @@ -185,7 +193,9 @@ public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path pat this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; + // TODO: Move this back to upper layer this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); + this.timeRangeTrackerForTiering = () -> this.timeRangeTracker; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); @@ -588,7 +598,8 @@ private BlockCacheKey buildCacheBlockKey(long offset, BlockType blockType) { } private boolean shouldCacheBlock(BlockCache cache, BlockCacheKey key) { - Optional result = cache.shouldCacheBlock(key, timeRangeTracker, conf); + Optional result = + cache.shouldCacheBlock(key, timeRangeTrackerForTiering.get().getMax(), conf); return result.orElse(true); } @@ -899,6 +910,13 @@ public void appendTrackedTimestampsToMetadata() throws IOException { appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); } + public void appendCustomCellTimestampsToMetadata(TimeRangeTracker timeRangeTracker) + throws IOException { + // TODO: The StoreFileReader always converts the byte[] to TimeRange + // via TimeRangeTracker, so we should write the serialization data of TimeRange directly. + appendFileInfo(CUSTOM_TIERING_TIME_RANGE, TimeRangeTracker.toByteArray(timeRangeTracker)); + } + /** * Record the earliest Put timestamp. If the timeRangeTracker is not set, update TimeRangeTracker * to include the timestamp of this key diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index f39032840678..8b333bce0b5a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -86,7 +86,6 @@ import org.apache.hadoop.hbase.regionserver.DataTieringManager; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.IdReadWriteLock; @@ -1116,8 +1115,9 @@ void freeSpace(final String why) { } } - if (bytesFreed < bytesToFreeWithExtra && - coldFiles != null && coldFiles.containsKey(bucketEntryWithKey.getKey().getHfileName()) + if ( + bytesFreed < bytesToFreeWithExtra && coldFiles != null + && coldFiles.containsKey(bucketEntryWithKey.getKey().getHfileName()) ) { int freedBlockSize = bucketEntryWithKey.getValue().getLength(); if (evictBlockIfNoRpcReferenced(bucketEntryWithKey.getKey())) { @@ -2365,10 +2365,10 @@ public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf } @Override - public Optional shouldCacheBlock(BlockCacheKey key, TimeRangeTracker timeRangeTracker, + public Optional shouldCacheBlock(BlockCacheKey key, long maxTimestamp, Configuration conf) { DataTieringManager dataTieringManager = DataTieringManager.getInstance(); - if (dataTieringManager != null && !dataTieringManager.isHotData(timeRangeTracker, conf)) { + if (dataTieringManager != null && !dataTieringManager.isHotData(maxTimestamp, conf)) { LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", key.getHfileName()); return Optional.of(false); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 67e0236c7be7..928b07ea5611 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.hadoop.hbase.procedure2.ProcedureUtil; +import org.apache.hadoop.hbase.regionserver.compactions.CustomCellTieredUtils; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils; import org.apache.hadoop.hbase.replication.ReplicationException; @@ -296,6 +297,8 @@ private boolean prepareCreate(final MasterProcedureEnv env) throws IOException { StoreFileTrackerValidationUtils.checkForCreateTable(env.getMasterConfiguration(), tableDescriptor); + CustomCellTieredUtils.checkForModifyTable(tableDescriptor); + return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index 5b51a5662db9..76f3c086bea1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; +import org.apache.hadoop.hbase.regionserver.compactions.CustomCellTieredUtils; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; @@ -449,6 +450,7 @@ private void prepareModify(final MasterProcedureEnv env) throws IOException { // check for store file tracker configurations StoreFileTrackerValidationUtils.checkForModifyTable(env.getMasterConfiguration(), unmodifiedTableDescriptor, modifiedTableDescriptor, !isTableEnabled(env)); + CustomCellTieredUtils.checkForModifyTable(modifiedTableDescriptor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellTSTiering.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellTSTiering.java new file mode 100644 index 000000000000..ed7dc01ba8d2 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellTSTiering.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; + +import java.io.IOException; +import java.util.OptionalLong; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public class CellTSTiering implements DataTiering { + private static final Logger LOG = LoggerFactory.getLogger(CellTSTiering.class); + + public long getTimestamp(HStoreFile hStoreFile) { + OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); + if (!maxTimestamp.isPresent()) { + LOG.debug("Maximum timestamp not present for {}", hStoreFile.getPath()); + return Long.MAX_VALUE; + } + return maxTimestamp.getAsLong(); + } + + public long getTimestamp(HFileInfo hFileInfo) { + try { + byte[] hFileTimeRange = hFileInfo.get(TIMERANGE_KEY); + if (hFileTimeRange == null) { + LOG.debug("Timestamp information not found for file: {}", + hFileInfo.getHFileContext().getHFileName()); + return Long.MAX_VALUE; + } + return TimeRangeTracker.parseFrom(hFileTimeRange).getMax(); + } catch (IOException e) { + LOG.error("Error occurred while reading the timestamp metadata of file: {}", + hFileInfo.getHFileContext().getHFileName(), e); + return Long.MAX_VALUE; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieredStoreEngine.java new file mode 100644 index 000000000000..518b31fb5be4 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieredStoreEngine.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CompoundConfiguration; +import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy; +import org.apache.hadoop.hbase.regionserver.compactions.CustomTieredCompactor; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Extension of {@link DateTieredStoreEngine} that uses a pluggable value provider for extracting + * the value to be used for comparison in this tiered compaction. Differently from the existing Date + * Tiered Compaction, this doesn't yield multiple tiers or files, but rather provides two tiers + * based on a configurable “cut-off” age. All rows with the cell tiering value older than this + * “cut-off” age would be placed together in an “old” tier, whilst younger rows would go to a + * separate, “young” tier file. + */ +@InterfaceAudience.Private +public class CustomTieredStoreEngine extends DateTieredStoreEngine { + + @Override + protected void createComponents(Configuration conf, HStore store, CellComparator kvComparator) + throws IOException { + CompoundConfiguration config = new CompoundConfiguration(); + config.add(conf); + config.add(store.conf); + config.set(DEFAULT_COMPACTION_POLICY_CLASS_KEY, + CustomDateTieredCompactionPolicy.class.getName()); + createCompactionPolicy(config, store); + this.storeFileManager = new DefaultStoreFileManager(kvComparator, + StoreFileComparators.SEQ_ID_MAX_TIMESTAMP, config, compactionPolicy.getConf()); + this.storeFlusher = new DefaultStoreFlusher(config, store); + this.compactor = new CustomTieredCompactor(config, store); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTiering.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTiering.java new file mode 100644 index 000000000000..7a9914c87d34 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTiering.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE; + +import java.io.IOException; +import java.util.Date; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public class CustomTiering implements DataTiering { + private static final Logger LOG = LoggerFactory.getLogger(CustomTiering.class); + + private long getMaxTSFromTimeRange(byte[] hFileTimeRange, String hFileName) { + try { + if (hFileTimeRange == null) { + LOG.debug("Custom cell-based timestamp information not found for file: {}", hFileName); + return Long.MAX_VALUE; + } + long parsedValue = TimeRangeTracker.parseFrom(hFileTimeRange).getMax(); + LOG.debug("Max TS for file {} is {}", hFileName, new Date(parsedValue)); + return parsedValue; + } catch (IOException e) { + LOG.error("Error occurred while reading the Custom cell-based timestamp metadata of file: {}", + hFileName, e); + return Long.MAX_VALUE; + } + } + + public long getTimestamp(HStoreFile hStoreFile) { + return getMaxTSFromTimeRange(hStoreFile.getMetadataValue(CUSTOM_TIERING_TIME_RANGE), + hStoreFile.getPath().getName()); + } + + public long getTimestamp(HFileInfo hFileInfo) { + return getMaxTSFromTimeRange(hFileInfo.get(CUSTOM_TIERING_TIME_RANGE), + hFileInfo.getHFileContext().getHFileName()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieringMultiFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieringMultiFileWriter.java new file mode 100644 index 000000000000..f2062fbf27b6 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomTieringMultiFileWriter.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.function.Function; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.io.hfile.HFileWriterImpl; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class CustomTieringMultiFileWriter extends DateTieredMultiFileWriter { + + public static final byte[] CUSTOM_TIERING_TIME_RANGE = Bytes.toBytes("CUSTOM_TIERING_TIME_RANGE"); + + private NavigableMap lowerBoundary2TimeRanger = new TreeMap<>(); + + public CustomTieringMultiFileWriter(List lowerBoundaries, + Map lowerBoundariesPolicies, boolean needEmptyFile, + Function tieringFunction) { + super(lowerBoundaries, lowerBoundariesPolicies, needEmptyFile, tieringFunction); + for (Long lowerBoundary : lowerBoundaries) { + lowerBoundary2TimeRanger.put(lowerBoundary, null); + } + } + + @Override + public void append(Cell cell) throws IOException { + super.append(cell); + long tieringValue = tieringFunction.apply(cell); + Map.Entry entry = lowerBoundary2TimeRanger.floorEntry(tieringValue); + if (entry.getValue() == null) { + TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); + timeRangeTracker.setMin(tieringValue); + timeRangeTracker.setMax(tieringValue); + lowerBoundary2TimeRanger.put(entry.getKey(), timeRangeTracker); + ((HFileWriterImpl) lowerBoundary2Writer.get(entry.getKey()).getLiveFileWriter()) + .setTimeRangeTrackerForTiering(() -> timeRangeTracker); + } else { + TimeRangeTracker timeRangeTracker = entry.getValue(); + if (timeRangeTracker.getMin() > tieringValue) { + timeRangeTracker.setMin(tieringValue); + } + if (timeRangeTracker.getMax() < tieringValue) { + timeRangeTracker.setMax(tieringValue); + } + } + } + + @Override + public List commitWriters(long maxSeqId, boolean majorCompaction, + Collection storeFiles) throws IOException { + for (Map.Entry entry : this.lowerBoundary2Writer.entrySet()) { + StoreFileWriter writer = entry.getValue(); + if (writer != null) { + writer.appendFileInfo(CUSTOM_TIERING_TIME_RANGE, + TimeRangeTracker.toByteArray(lowerBoundary2TimeRanger.get(entry.getKey()))); + } + } + return super.commitWriters(maxSeqId, majorCompaction, storeFiles); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTiering.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTiering.java new file mode 100644 index 000000000000..51e89b0b79d0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTiering.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public interface DataTiering { + long getTimestamp(HStoreFile hFile); + + long getTimestamp(HFileInfo hFileInfo); + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index f71bc5e43aa6..8443827ccaa1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -17,13 +17,11 @@ */ package org.apache.hadoop.hbase.regionserver; -import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; - import java.io.IOException; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.OptionalLong; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -136,17 +134,18 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException { * the data tiering type is set to {@link DataTieringType#TIME_RANGE}, it uses the maximum * timestamp from the time range tracker to determine if the data is hot. Otherwise, it considers * the data as hot by default. - * @param timeRangeTracker the time range tracker containing the timestamps - * @param conf The configuration object to use for determining hot data criteria. + * @param maxTimestamp the maximum timestamp associated with the data. + * @param conf The configuration object to use for determining hot data criteria. * @return {@code true} if the data is hot, {@code false} otherwise */ - public boolean isHotData(TimeRangeTracker timeRangeTracker, Configuration conf) { + public boolean isHotData(long maxTimestamp, Configuration conf) { DataTieringType dataTieringType = getDataTieringType(conf); + if ( - dataTieringType.equals(DataTieringType.TIME_RANGE) - && timeRangeTracker.getMax() != TimeRangeTracker.INITIAL_MAX_TIMESTAMP + !dataTieringType.equals(DataTieringType.NONE) + && maxTimestamp != TimeRangeTracker.INITIAL_MAX_TIMESTAMP ) { - return hotDataValidator(timeRangeTracker.getMax(), getDataTieringHotDataAge(conf)); + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(conf)); } // DataTieringType.NONE or other types are considered hot by default return true; @@ -165,29 +164,14 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { Configuration configuration = getConfiguration(hFilePath); DataTieringType dataTieringType = getDataTieringType(configuration); - if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { - return hotDataValidator(getMaxTimestamp(hFilePath), getDataTieringHotDataAge(configuration)); - } - // DataTieringType.NONE or other types are considered hot by default - return true; - } - - /** - * Determines whether the data in the HFile at the given path is considered hot based on the - * configured data tiering type and hot data age. If the data tiering type is set to - * {@link DataTieringType#TIME_RANGE}, it validates the data against the provided maximum - * timestamp. - * @param hFilePath the path to the HFile - * @param maxTimestamp the maximum timestamp to validate against - * @return {@code true} if the data is hot, {@code false} otherwise - * @throws DataTieringException if there is an error retrieving data tiering information - */ - public boolean isHotData(Path hFilePath, long maxTimestamp) throws DataTieringException { - Configuration configuration = getConfiguration(hFilePath); - DataTieringType dataTieringType = getDataTieringType(configuration); - - if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { - return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); + if (!dataTieringType.equals(DataTieringType.NONE)) { + HStoreFile hStoreFile = getHStoreFile(hFilePath); + if (hStoreFile == null) { + throw new DataTieringException( + "Store file corresponding to " + hFilePath + " doesn't exist"); + } + return hotDataValidator(dataTieringType.getInstance().getTimestamp(getHStoreFile(hFilePath)), + getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; @@ -204,8 +188,9 @@ public boolean isHotData(Path hFilePath, long maxTimestamp) throws DataTieringEx */ public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) { DataTieringType dataTieringType = getDataTieringType(configuration); - if (dataTieringType.equals(DataTieringType.TIME_RANGE)) { - return hotDataValidator(getMaxTimestamp(hFileInfo), getDataTieringHotDataAge(configuration)); + if (hFileInfo != null && !dataTieringType.equals(DataTieringType.NONE)) { + return hotDataValidator(dataTieringType.getInstance().getTimestamp(hFileInfo), + getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; @@ -217,36 +202,6 @@ private boolean hotDataValidator(long maxTimestamp, long hotDataAge) { return diff <= hotDataAge; } - private long getMaxTimestamp(Path hFilePath) throws DataTieringException { - HStoreFile hStoreFile = getHStoreFile(hFilePath); - if (hStoreFile == null) { - LOG.error("HStoreFile corresponding to {} doesn't exist", hFilePath); - return Long.MAX_VALUE; - } - OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); - if (!maxTimestamp.isPresent()) { - LOG.error("Maximum timestamp not present for {}", hFilePath); - return Long.MAX_VALUE; - } - return maxTimestamp.getAsLong(); - } - - private long getMaxTimestamp(HFileInfo hFileInfo) { - try { - byte[] hFileTimeRange = hFileInfo.get(TIMERANGE_KEY); - if (hFileTimeRange == null) { - LOG.error("Timestamp information not found for file: {}", - hFileInfo.getHFileContext().getHFileName()); - return Long.MAX_VALUE; - } - return TimeRangeTracker.parseFrom(hFileTimeRange).getMax(); - } catch (IOException e) { - LOG.error("Error occurred while reading the timestamp metadata of file: {}", - hFileInfo.getHFileContext().getHFileName(), e); - return Long.MAX_VALUE; - } - } - private long getCurrentTimestamp() { return EnvironmentEdgeManager.getDelegate().currentTime(); } @@ -299,7 +254,7 @@ private HStore getHStore(Path hFilePath) throws DataTieringException { private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException { HStore hStore = getHStore(hFilePath); for (HStoreFile file : hStore.getStorefiles()) { - if (file.getPath().toUri().getPath().toString().equals(hFilePath.toString())) { + if (file.getPath().equals(hFilePath)) { return file; } } @@ -330,7 +285,8 @@ public Map getColdFilesList() { for (HRegion r : this.onlineRegions.values()) { for (HStore hStore : r.getStores()) { Configuration conf = hStore.getReadOnlyConfiguration(); - if (getDataTieringType(conf) != DataTieringType.TIME_RANGE) { + DataTieringType dataTieringType = getDataTieringType(conf); + if (dataTieringType == DataTieringType.NONE) { // Data-Tiering not enabled for the store. Just skip it. continue; } @@ -339,14 +295,10 @@ public Map getColdFilesList() { for (HStoreFile hStoreFile : hStore.getStorefiles()) { String hFileName = hStoreFile.getFileInfo().getHFileInfo().getHFileContext().getHFileName(); - OptionalLong maxTimestamp = hStoreFile.getMaximumTimestamp(); - if (!maxTimestamp.isPresent()) { - LOG.warn("maxTimestamp missing for file: {}", - hStoreFile.getFileInfo().getActiveFileName()); - continue; - } + long maxTimeStamp = dataTieringType.getInstance().getTimestamp(hStoreFile); + LOG.debug("Max TS for file {} is {}", hFileName, new Date(maxTimeStamp)); long currentTimestamp = EnvironmentEdgeManager.getDelegate().currentTime(); - long fileAge = currentTimestamp - maxTimestamp.getAsLong(); + long fileAge = currentTimestamp - maxTimeStamp; if (fileAge > hotDataAge) { // Values do not matter. coldFiles.put(hFileName, null); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java index ee54576a6487..83da5f54e43f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringType.java @@ -21,6 +21,17 @@ @InterfaceAudience.Public public enum DataTieringType { - NONE, - TIME_RANGE + NONE(null), + TIME_RANGE(new CellTSTiering()), + CUSTOM(new CustomTiering()); + + private final DataTiering instance; + + DataTieringType(DataTiering instance) { + this.instance = instance; + } + + public DataTiering getInstance() { + return instance; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredMultiFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredMultiFileWriter.java index e5ee8041c350..828d9ba00101 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredMultiFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredMultiFileWriter.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import java.util.function.Function; import org.apache.hadoop.hbase.Cell; import org.apache.yetus.audience.InterfaceAudience; @@ -33,12 +34,14 @@ @InterfaceAudience.Private public class DateTieredMultiFileWriter extends AbstractMultiFileWriter { - private final NavigableMap lowerBoundary2Writer = new TreeMap<>(); + protected final NavigableMap lowerBoundary2Writer = new TreeMap<>(); private final boolean needEmptyFile; private final Map lowerBoundariesPolicies; + protected Function tieringFunction; + /** * @param lowerBoundariesPolicies each window to storage policy map. * @param needEmptyFile whether need to create an empty store file if we haven't written @@ -46,16 +49,29 @@ public class DateTieredMultiFileWriter extends AbstractMultiFileWriter { */ public DateTieredMultiFileWriter(List lowerBoundaries, Map lowerBoundariesPolicies, boolean needEmptyFile) { + this(lowerBoundaries, lowerBoundariesPolicies, needEmptyFile, c -> c.getTimestamp()); + } + + /** + * @param lowerBoundariesPolicies each window to storage policy map. + * @param needEmptyFile whether need to create an empty store file if we haven't written + * out anything. + */ + public DateTieredMultiFileWriter(List lowerBoundaries, + Map lowerBoundariesPolicies, boolean needEmptyFile, + Function tieringFunction) { for (Long lowerBoundary : lowerBoundaries) { lowerBoundary2Writer.put(lowerBoundary, null); } this.needEmptyFile = needEmptyFile; this.lowerBoundariesPolicies = lowerBoundariesPolicies; + this.tieringFunction = tieringFunction; } @Override public void append(Cell cell) throws IOException { - Map.Entry entry = lowerBoundary2Writer.floorEntry(cell.getTimestamp()); + Map.Entry entry = + lowerBoundary2Writer.floorEntry(tieringFunction.apply(cell)); StoreFileWriter writer = entry.getValue(); if (writer == null) { String lowerBoundaryStoragePolicy = lowerBoundariesPolicies.get(entry.getKey()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 26437ab11242..dc13f190afaa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.regionserver.DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY; + import java.io.IOException; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -29,6 +31,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactor; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; /** @@ -44,6 +47,18 @@ public class DateTieredStoreEngine extends StoreEngine filesCompacting) { return compactionPolicy.needsCompaction(storeFileManager.getStoreFiles(), filesCompacting); @@ -57,7 +72,7 @@ public CompactionContext createCompaction() throws IOException { @Override protected void createComponents(Configuration conf, HStore store, CellComparator kvComparator) throws IOException { - this.compactionPolicy = new DateTieredCompactionPolicy(conf, store); + createCompactionPolicy(conf, store); this.storeFileManager = new DefaultStoreFileManager(kvComparator, StoreFileComparators.SEQ_ID_MAX_TIMESTAMP, conf, compactionPolicy.getConf()); this.storeFlusher = new DefaultStoreFlusher(conf, store); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 8fa555cc088d..4085315ea882 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -701,8 +701,8 @@ public HRegionServer(final Configuration conf) throws IOException { if (!isMasterNotCarryTable) { blockCache = BlockCacheFactory.createBlockCache(conf); // The call below, instantiates the DataTieringManager only when - // the configuration "hbase.regionserver.datatiering.enable" is set to true. - DataTieringManager.instantiate(conf,onlineRegions); + // the configuration "hbase.regionserver.datatiering.enable" is set to true. + DataTieringManager.instantiate(conf, onlineRegions); mobFileCache = new MobFileCache(conf); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java index 5f5fcf2001a0..8a705c5ef147 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java @@ -255,6 +255,14 @@ public void appendTrackedTimestampsToMetadata() throws IOException { } } + public void appendCustomCellTimestampsToMetadata(TimeRangeTracker timeRangeTracker) + throws IOException { + liveFileWriter.appendCustomCellTimestampsToMetadata(timeRangeTracker); + if (historicalFileWriter != null) { + historicalFileWriter.appendCustomCellTimestampsToMetadata(timeRangeTracker); + } + } + @Override public void beforeShipped() throws IOException { liveFileWriter.beforeShipped(); @@ -664,6 +672,11 @@ private void appendTrackedTimestampsToMetadata() throws IOException { writer.appendTrackedTimestampsToMetadata(); } + public void appendCustomCellTimestampsToMetadata(TimeRangeTracker timeRangeTracker) + throws IOException { + writer.appendCustomCellTimestampsToMetadata(timeRangeTracker); + } + private void appendGeneralBloomfilter(final Cell cell) throws IOException { if (this.generalBloomFilterWriter != null) { /* diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index 3e9e85a4aba8..f18abf59309d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -401,6 +401,11 @@ protected abstract List commitWriter(T writer, FileDetails fd, protected abstract void abortWriter(T writer) throws IOException; + protected List decorateCells(List cells) { + // no op + return cells; + } + /** * Performs the compaction. * @param fd FileDetails of cell sink writer @@ -454,6 +459,7 @@ protected boolean performCompaction(FileDetails fd, InternalScanner scanner, Cel // output to writer: Cell lastCleanCell = null; long lastCleanCellSeqId = 0; + cells = decorateCells(cells); for (Cell c : cells) { if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) { lastCleanCell = c; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieredUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieredUtils.java new file mode 100644 index 000000000000..f908b31e4ae5 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieredUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import static org.apache.hadoop.hbase.regionserver.StoreEngine.STORE_ENGINE_CLASS_KEY; +import static org.apache.hadoop.hbase.regionserver.compactions.CustomCellTieringValueProvider.TIERING_CELL_QUALIFIER; + +import java.io.IOException; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class CustomCellTieredUtils { + private CustomCellTieredUtils() { + // Utility class, no instantiation + } + + public static void checkForModifyTable(TableDescriptor newTable) throws IOException { + for (ColumnFamilyDescriptor descriptor : newTable.getColumnFamilies()) { + String storeEngineClassName = descriptor.getConfigurationValue(STORE_ENGINE_CLASS_KEY); + if ( + storeEngineClassName != null && storeEngineClassName.contains("CustomCellTieredStoreEngine") + ) { + if (descriptor.getConfigurationValue(TIERING_CELL_QUALIFIER) == null) { + throw new DoNotRetryIOException("StoreEngine " + storeEngineClassName + + " is missing required " + TIERING_CELL_QUALIFIER + " parameter."); + } + } + } + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieringValueProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieringValueProvider.java new file mode 100644 index 000000000000..fcf3b203b10f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomCellTieringValueProvider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ArrayBackedTag; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.TagType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An extension of DateTieredCompactor, overriding the decorateCells method to allow for custom + * values to be used for the different file tiers during compaction. + */ +@InterfaceAudience.Private +public class CustomCellTieringValueProvider implements CustomTieredCompactor.TieringValueProvider { + public static final String TIERING_CELL_QUALIFIER = "TIERING_CELL_QUALIFIER"; + private byte[] tieringQualifier; + + @Override + public void init(Configuration conf) throws Exception { + tieringQualifier = Bytes.toBytes(conf.get(TIERING_CELL_QUALIFIER)); + } + + @Override + public List decorateCells(List cells) { + // if no tiering qualifier properly set, skips the whole flow + if (tieringQualifier != null) { + byte[] tieringValue = null; + // first iterates through the cells within a row, to find the tiering value for the row + for (Cell cell : cells) { + if (CellUtil.matchingQualifier(cell, tieringQualifier)) { + tieringValue = CellUtil.cloneValue(cell); + break; + } + } + if (tieringValue == null) { + tieringValue = Bytes.toBytes(Long.MAX_VALUE); + } + // now apply the tiering value as a tag to all cells within the row + Tag tieringValueTag = new ArrayBackedTag(TagType.CELL_VALUE_TIERING_TAG_TYPE, tieringValue); + List newCells = new ArrayList<>(cells.size()); + for (Cell cell : cells) { + List tags = PrivateCellUtil.getTags(cell); + tags.add(tieringValueTag); + newCells.add(PrivateCellUtil.createCell(cell, tags)); + } + return newCells; + } else { + return cells; + } + } + + @Override + public long getTieringValue(Cell cell) { + Optional tagOptional = PrivateCellUtil.getTag(cell, TagType.CELL_VALUE_TIERING_TAG_TYPE); + if (tagOptional.isPresent()) { + Tag tag = tagOptional.get(); + return Bytes.toLong(tag.getValueByteBuffer().array(), tag.getValueOffset(), + tag.getValueLength()); + } + return Long.MAX_VALUE; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomDateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomDateTieredCompactionPolicy.java new file mode 100644 index 000000000000..dcc97c63d024 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomDateTieredCompactionPolicy.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreConfigInformation; +import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Custom implementation of DateTieredCompactionPolicy that calculates compaction boundaries based + * on the hbase.hstore.compaction.date.tiered.custom.age.limit.millis configuration property + * and the TIERING_CELL_MIN/TIERING_CELL_MAX stored on metadata of each store file. This policy + * would produce either one or two tiers: - One tier if either all files data age are older than the + * configured age limit or all files data age are younger than the configured age limit. - Two tiers + * if files have both younger and older data than the configured age limit. + */ +@InterfaceAudience.Private +public class CustomDateTieredCompactionPolicy extends DateTieredCompactionPolicy { + + public static final String AGE_LIMIT_MILLIS = + "hbase.hstore.compaction.date.tiered.custom.age.limit.millis"; + + // Defaults to 10 years + public static final long DEFAULT_AGE_LIMIT_MILLIS = + (long) (10L * 365.25 * 24L * 60L * 60L * 1000L); + + private static final Logger LOG = LoggerFactory.getLogger(CustomDateTieredCompactionPolicy.class); + + private long cutOffTimestamp; + + public CustomDateTieredCompactionPolicy(Configuration conf, + StoreConfigInformation storeConfigInfo) throws IOException { + super(conf, storeConfigInfo); + cutOffTimestamp = EnvironmentEdgeManager.currentTime() + - conf.getLong(AGE_LIMIT_MILLIS, DEFAULT_AGE_LIMIT_MILLIS); + + } + + @Override + protected List getCompactBoundariesForMajor(Collection filesToCompact, + long now) { + MutableLong min = new MutableLong(Long.MAX_VALUE); + MutableLong max = new MutableLong(0); + filesToCompact.forEach(f -> { + byte[] timeRangeBytes = f.getMetadataValue(CUSTOM_TIERING_TIME_RANGE); + long minCurrent = Long.MAX_VALUE; + long maxCurrent = 0; + if (timeRangeBytes != null) { + try { + TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(timeRangeBytes); + timeRangeTracker.getMin(); + minCurrent = timeRangeTracker.getMin(); + maxCurrent = timeRangeTracker.getMax(); + } catch (IOException e) { + LOG.warn("Got TIERING_CELL_TIME_RANGE info from file, but failed to parse it:", e); + } + } + if (minCurrent < min.getValue()) { + min.setValue(minCurrent); + } + if (maxCurrent > max.getValue()) { + max.setValue(maxCurrent); + } + }); + + List boundaries = new ArrayList<>(); + boundaries.add(Long.MIN_VALUE); + if (min.getValue() < cutOffTimestamp) { + boundaries.add(min.getValue()); + if (max.getValue() > cutOffTimestamp) { + boundaries.add(cutOffTimestamp); + } + } + return boundaries; + } + + @Override + public CompactionRequestImpl selectMinorCompaction(ArrayList candidateSelection, + boolean mayUseOffPeak, boolean mayBeStuck) throws IOException { + ArrayList filteredByPolicy = this.compactionPolicyPerWindow + .applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck); + return selectMajorCompaction(filteredByPolicy); + } + + @Override + public boolean shouldPerformMajorCompaction(Collection filesToCompact) + throws IOException { + long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); + long now = EnvironmentEdgeManager.currentTime(); + if (isMajorCompactionTime(filesToCompact, now, lowTimestamp)) { + long cfTTL = this.storeConfigInfo.getStoreFileTtl(); + int countLower = 0; + int countHigher = 0; + HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); + for (HStoreFile f : filesToCompact) { + if (checkForTtl(cfTTL, f)) { + return true; + } + if (isMajorOrBulkloadResult(f, now - lowTimestamp)) { + return true; + } + byte[] timeRangeBytes = f.getMetadataValue(CUSTOM_TIERING_TIME_RANGE); + TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(timeRangeBytes); + if (timeRangeTracker.getMin() < cutOffTimestamp) { + if (timeRangeTracker.getMax() > cutOffTimestamp) { + // Found at least one file crossing the cutOffTimestamp + return true; + } else { + countLower++; + } + } else { + countHigher++; + } + hdfsBlocksDistribution.add(f.getHDFSBlockDistribution()); + } + // If we haven't found any files crossing the cutOffTimestamp, we have to check + // if there are at least more than one file on each tier and if so, perform compaction + if (countLower > 1 || countHigher > 1) { + return true; + } + return checkBlockLocality(hdfsBlocksDistribution); + } + return false; + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomTieredCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomTieredCompactor.java new file mode 100644 index 000000000000..905284a19c4a --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CustomTieredCompactor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter; +import org.apache.hadoop.hbase.regionserver.DateTieredMultiFileWriter; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class CustomTieredCompactor extends DateTieredCompactor { + + public static final String TIERING_VALUE_PROVIDER = + "hbase.hstore.custom-tiering-value.provider.class"; + private TieringValueProvider tieringValueProvider; + + public CustomTieredCompactor(Configuration conf, HStore store) throws IOException { + super(conf, store); + String className = + conf.get(TIERING_VALUE_PROVIDER, CustomCellTieringValueProvider.class.getName()); + try { + tieringValueProvider = + (TieringValueProvider) Class.forName(className).getConstructor().newInstance(); + tieringValueProvider.init(conf); + } catch (Exception e) { + throw new IOException("Unable to load configured tiering value provider '" + className + "'", + e); + } + } + + @Override + protected List decorateCells(List cells) { + return tieringValueProvider.decorateCells(cells); + } + + @Override + protected DateTieredMultiFileWriter createMultiWriter(final CompactionRequestImpl request, + final List lowerBoundaries, final Map lowerBoundariesPolicies) { + return new CustomTieringMultiFileWriter(lowerBoundaries, lowerBoundariesPolicies, + needEmptyFile(request), CustomTieredCompactor.this.tieringValueProvider::getTieringValue); + } + + public interface TieringValueProvider { + + void init(Configuration configuration) throws Exception; + + default List decorateCells(List cells) { + return cells; + } + + long getTieringValue(Cell cell); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java index c13bcf36af92..81cfd6a0c69a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver.compactions; import java.io.IOException; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -66,7 +67,7 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy { private static final Logger LOG = LoggerFactory.getLogger(DateTieredCompactionPolicy.class); - private final RatioBasedCompactionPolicy compactionPolicyPerWindow; + protected final RatioBasedCompactionPolicy compactionPolicyPerWindow; private final CompactionWindowFactory windowFactory; @@ -108,9 +109,8 @@ public boolean needsCompaction(Collection storeFiles, } } - @Override - public boolean shouldPerformMajorCompaction(Collection filesToCompact) - throws IOException { + protected boolean isMajorCompactionTime(Collection filesToCompact, long now, + long lowestModificationTime) throws IOException { long mcTime = getNextMajorCompactTime(filesToCompact); if (filesToCompact == null || mcTime == 0) { if (LOG.isDebugEnabled()) { @@ -118,58 +118,40 @@ public boolean shouldPerformMajorCompaction(Collection filesToCompac } return false; } - // TODO: Use better method for determining stamp of last major (HBASE-2990) - long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); - long now = EnvironmentEdgeManager.currentTime(); - if (lowTimestamp <= 0L || lowTimestamp >= (now - mcTime)) { + if (lowestModificationTime <= 0L || lowestModificationTime >= (now - mcTime)) { if (LOG.isDebugEnabled()) { - LOG.debug("lowTimestamp: " + lowTimestamp + " lowTimestamp: " + lowTimestamp + " now: " - + now + " mcTime: " + mcTime); + LOG.debug("lowTimestamp: " + lowestModificationTime + " lowTimestamp: " + + lowestModificationTime + " now: " + now + " mcTime: " + mcTime); } return false; } + return true; + } - long cfTTL = this.storeConfigInfo.getStoreFileTtl(); - HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); - List boundaries = getCompactBoundariesForMajor(filesToCompact, now); - boolean[] filesInWindow = new boolean[boundaries.size()]; - - for (HStoreFile file : filesToCompact) { - OptionalLong minTimestamp = file.getMinimumTimestamp(); - long oldest = minTimestamp.isPresent() ? now - minTimestamp.getAsLong() : Long.MIN_VALUE; - if (cfTTL != Long.MAX_VALUE && oldest >= cfTTL) { - LOG.debug("Major compaction triggered on store " + this + "; for TTL maintenance"); - return true; - } - if (!file.isMajorCompactionResult() || file.isBulkLoadResult()) { - LOG.debug("Major compaction triggered on store " + this - + ", because there are new files and time since last major compaction " - + (now - lowTimestamp) + "ms"); - return true; - } + protected boolean checkForTtl(long ttl, HStoreFile file) { + OptionalLong minTimestamp = file.getMinimumTimestamp(); + long oldest = minTimestamp.isPresent() + ? EnvironmentEdgeManager.currentTime() - minTimestamp.getAsLong() + : Long.MIN_VALUE; + if (ttl != Long.MAX_VALUE && oldest >= ttl) { + LOG.debug("Major compaction triggered on store " + this + "; for TTL maintenance"); + return true; + } + return false; + } - int lowerWindowIndex = - Collections.binarySearch(boundaries, minTimestamp.orElse(Long.MAX_VALUE)); - int upperWindowIndex = - Collections.binarySearch(boundaries, file.getMaximumTimestamp().orElse(Long.MAX_VALUE)); - // Handle boundary conditions and negative values of binarySearch - lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex; - upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex; - if (lowerWindowIndex != upperWindowIndex) { - LOG.debug("Major compaction triggered on store " + this + "; because file " + file.getPath() - + " has data with timestamps cross window boundaries"); - return true; - } else if (filesInWindow[upperWindowIndex]) { - LOG.debug("Major compaction triggered on store " + this - + "; because there are more than one file in some windows"); - return true; - } else { - filesInWindow[upperWindowIndex] = true; - } - hdfsBlocksDistribution.add(file.getHDFSBlockDistribution()); + protected boolean isMajorOrBulkloadResult(HStoreFile file, long timeDiff) { + if (!file.isMajorCompactionResult() || file.isBulkLoadResult()) { + LOG.debug("Major compaction triggered on store " + this + + ", because there are new files and time since last major compaction " + timeDiff + "ms"); + return true; } + return false; + } + protected boolean checkBlockLocality(HDFSBlocksDistribution hdfsBlocksDistribution) + throws UnknownHostException { float blockLocalityIndex = hdfsBlocksDistribution .getBlockLocalityIndex(DNS.getHostname(comConf.conf, DNS.ServerType.REGIONSERVER)); if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) { @@ -178,9 +160,55 @@ public boolean shouldPerformMajorCompaction(Collection filesToCompac + " (min " + comConf.getMinLocalityToForceCompact() + ")"); return true; } + return false; + } - LOG.debug( - "Skipping major compaction of " + this + ", because the files are already major compacted"); + @Override + public boolean shouldPerformMajorCompaction(Collection filesToCompact) + throws IOException { + long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact); + long now = EnvironmentEdgeManager.currentTime(); + if (isMajorCompactionTime(filesToCompact, now, lowTimestamp)) { + long cfTTL = this.storeConfigInfo.getStoreFileTtl(); + HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); + List boundaries = getCompactBoundariesForMajor(filesToCompact, now); + boolean[] filesInWindow = new boolean[boundaries.size()]; + for (HStoreFile file : filesToCompact) { + OptionalLong minTimestamp = file.getMinimumTimestamp(); + if (checkForTtl(cfTTL, file)) { + return true; + } + if (isMajorOrBulkloadResult(file, now - lowTimestamp)) { + return true; + } + int lowerWindowIndex = + Collections.binarySearch(boundaries, minTimestamp.orElse(Long.MAX_VALUE)); + int upperWindowIndex = + Collections.binarySearch(boundaries, file.getMaximumTimestamp().orElse(Long.MAX_VALUE)); + // Handle boundary conditions and negative values of binarySearch + lowerWindowIndex = + (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex; + upperWindowIndex = + (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex; + if (lowerWindowIndex != upperWindowIndex) { + LOG.debug("Major compaction triggered on store " + this + "; because file " + + file.getPath() + " has data with timestamps cross window boundaries"); + return true; + } else if (filesInWindow[upperWindowIndex]) { + LOG.debug("Major compaction triggered on store " + this + + "; because there are more than one file in some windows"); + return true; + } else { + filesInWindow[upperWindowIndex] = true; + } + hdfsBlocksDistribution.add(file.getHDFSBlockDistribution()); + } + if (checkBlockLocality(hdfsBlocksDistribution)) { + return true; + } + LOG.debug( + "Skipping major compaction of " + this + ", because the files are already major compacted"); + } return false; } @@ -296,7 +324,8 @@ private DateTieredCompactionRequest generateCompactionRequest(ArrayList getCompactBoundariesForMajor(Collection filesToCompact, long now) { + protected List getCompactBoundariesForMajor(Collection filesToCompact, + long now) { long minTimestamp = filesToCompact.stream() .mapToLong(f -> f.getMinimumTimestamp().orElse(Long.MAX_VALUE)).min().orElse(Long.MAX_VALUE); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactor.java index b5911b0cec46..9cef2ebc3144 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactor.java @@ -46,7 +46,7 @@ public DateTieredCompactor(Configuration conf, HStore store) { super(conf, store); } - private boolean needEmptyFile(CompactionRequestImpl request) { + protected boolean needEmptyFile(CompactionRequestImpl request) { // if we are going to compact the last N files, then we need to emit an empty file to retain the // maxSeqId if we haven't written out anything. OptionalLong maxSeqId = StoreUtils.getMaxSequenceIdInList(request.getFiles()); @@ -70,14 +70,20 @@ public List compact(final CompactionRequestImpl request, final List public DateTieredMultiFileWriter createWriter(InternalScanner scanner, FileDetails fd, boolean shouldDropBehind, boolean major, Consumer writerCreationTracker) throws IOException { - DateTieredMultiFileWriter writer = new DateTieredMultiFileWriter(lowerBoundaries, - lowerBoundariesPolicies, needEmptyFile(request)); + DateTieredMultiFileWriter writer = + createMultiWriter(request, lowerBoundaries, lowerBoundariesPolicies); initMultiWriter(writer, scanner, fd, shouldDropBehind, major, writerCreationTracker); return writer; } }, throughputController, user); } + protected DateTieredMultiFileWriter createMultiWriter(final CompactionRequestImpl request, + final List lowerBoundaries, final Map lowerBoundariesPolicies) { + return new DateTieredMultiFileWriter(lowerBoundaries, lowerBoundariesPolicies, + needEmptyFile(request), c -> c.getTimestamp()); + } + @Override protected List commitWriter(DateTieredMultiFileWriter writer, FileDetails fd, CompactionRequestImpl request) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java index 2d45c05324be..d7c743fbf595 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java @@ -194,9 +194,10 @@ public void testIllegalTableDescriptor() throws Exception { @Test public void testIllegalTableDescriptorWithDataTiering() throws IOException { - // table level configuration changes HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); + // table level configuration changes + htd.addFamily(hcd); // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine htd.setValue(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name()); @@ -212,20 +213,23 @@ public void testIllegalTableDescriptorWithDataTiering() throws IOException { "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine"); checkTableIsIllegal(htd); + // column family level configuration changes + htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + hcd = new HColumnDescriptor(FAMILY); + // First scenario: DataTieringType set to TIME_RANGE without DateTieredStoreEngine - hcd.setConfiguration(DataTieringManager.DATATIERING_KEY, - DataTieringType.TIME_RANGE.name()); + hcd.setConfiguration(DataTieringManager.DATATIERING_KEY, DataTieringType.TIME_RANGE.name()); checkTableIsIllegal(htd.addFamily(hcd)); // Second scenario: DataTieringType set to TIME_RANGE with DateTieredStoreEngine hcd.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY, "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine"); - checkTableIsLegal(htd.addFamily(hcd)); + checkTableIsLegal(htd.modifyFamily(hcd)); // Third scenario: Disabling DateTieredStoreEngine while Time Range DataTiering is active hcd.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY, "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine"); - checkTableIsIllegal(htd.addFamily(hcd)); + checkTableIsIllegal(htd.modifyFamily(hcd)); } private void checkTableIsLegal(HTableDescriptor htd) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java index c85a162ad96a..7e7b4cb5c37c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java @@ -268,6 +268,10 @@ private void readLoadOnOpenDataSection(Path path, boolean hasBloomFilters) throw CacheConfig cacheConfig = new CacheConfig(conf); HFile.Reader reader = new HFilePreadReader(readerContext, hfile, cacheConfig, conf); + // Since HBASE-28466, we call fileInfo.initMetaAndIndex inside HFilePreadReader, + // which reads some blocks and increment the counters, so we need to reset it here. + ThreadLocalServerSideScanMetrics.getBytesReadFromFsAndReset(); + ThreadLocalServerSideScanMetrics.getBlockReadOpsCountAndReset(); HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); // Create iterator for reading root index block diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java new file mode 100644 index 000000000000..0771d41bb433 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java @@ -0,0 +1,860 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY; +import static org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.DEFAULT_ERROR_TOLERATION_DURATION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.BlockType; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is used to test the functionality of the DataTieringManager. + * + * The mock online regions are stored in {@link TestCustomCellDataTieringManager#testOnlineRegions}. + * For all tests, the setup of + * {@link TestCustomCellDataTieringManager#testOnlineRegions} occurs only once. + * Please refer to {@link TestCustomCellDataTieringManager#setupOnlineRegions()} for the structure. + * Additionally, a list of all store files is + * maintained in {@link TestCustomCellDataTieringManager#hStoreFiles}. + * The characteristics of these store files are listed below: + * @formatter:off + * ## HStoreFile Information + * | HStoreFile | Region | Store | DataTiering | isHot | + * |------------------|--------------------|---------------------|-----------------------|-------| + * | hStoreFile0 | region1 | hStore11 | CUSTOM_CELL_VALUE | true | + * | hStoreFile1 | region1 | hStore12 | NONE | true | + * | hStoreFile2 | region2 | hStore21 | CUSTOM_CELL_VALUE | true | + * | hStoreFile3 | region2 | hStore22 | CUSTOM_CELL_VALUE | false | + * @formatter:on + */ + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestCustomCellDataTieringManager { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCustomCellDataTieringManager.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestCustomCellDataTieringManager.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final long DAY = 24 * 60 * 60 * 1000; + private static Configuration defaultConf; + private static FileSystem fs; + private BlockCache blockCache; + private static CacheConfig cacheConf; + private static Path testDir; + private static final Map testOnlineRegions = new HashMap<>(); + + private static DataTieringManager dataTieringManager; + private static final List hStoreFiles = new ArrayList<>(); + + /** + * Represents the current lexicographically increasing string used as a row key when writing + * HFiles. It is incremented each time {@link #nextString()} is called to generate unique row + * keys. + */ + private static String rowKeyString; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + testDir = TEST_UTIL.getDataTestDir(TestCustomCellDataTieringManager.class.getSimpleName()); + defaultConf = TEST_UTIL.getConfiguration(); + updateCommonConfigurations(); + DataTieringManager.instantiate(defaultConf, testOnlineRegions); + dataTieringManager = DataTieringManager.getInstance(); + rowKeyString = ""; + } + + private static void updateCommonConfigurations() { + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); + defaultConf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + defaultConf.setLong(BUCKET_CACHE_SIZE_KEY, 32); + } + + @FunctionalInterface + interface DataTieringMethodCallerWithPath { + boolean call(DataTieringManager manager, Path path) throws DataTieringException; + } + + @FunctionalInterface + interface DataTieringMethodCallerWithKey { + boolean call(DataTieringManager manager, BlockCacheKey key) throws DataTieringException; + } + + @Test + public void testDataTieringEnabledWithKey() throws IOException { + initializeTestEnvironment(); + DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isDataTieringEnabled; + + // Test with valid key + BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true); + + // Test with another valid key + key = new BlockCacheKey(hStoreFiles.get(1).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false); + + // Test with valid key with no HFile Path + key = new BlockCacheKey(hStoreFiles.get(0).getPath().getName(), 0); + testDataTieringMethodWithKeyExpectingException(methodCallerWithKey, key, + new DataTieringException("BlockCacheKey Doesn't Contain HFile Path")); + } + + @Test + public void testDataTieringEnabledWithPath() throws IOException { + initializeTestEnvironment(); + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isDataTieringEnabled; + + // Test with valid path + Path hFilePath = hStoreFiles.get(1).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Test with another valid path + hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + + // Test with an incorrect path + hFilePath = new Path("incorrectPath"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("Incorrect HFile Path: " + hFilePath)); + + // Test with a non-existing HRegion path + Path basePath = hStoreFiles.get(0).getPath().getParent().getParent().getParent(); + hFilePath = new Path(basePath, "incorrectRegion/cf1/filename"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("HRegion corresponding to " + hFilePath + " doesn't exist")); + + // Test with a non-existing HStore path + basePath = hStoreFiles.get(0).getPath().getParent().getParent(); + hFilePath = new Path(basePath, "incorrectCf/filename"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("HStore corresponding to " + hFilePath + " doesn't exist")); + } + + @Test + public void testHotDataWithKey() throws IOException { + initializeTestEnvironment(); + DataTieringMethodCallerWithKey methodCallerWithKey = DataTieringManager::isHotData; + + // Test with valid key + BlockCacheKey key = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, true); + + // Test with another valid key + key = new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA); + testDataTieringMethodWithKeyNoException(methodCallerWithKey, key, false); + } + + @Test + public void testHotDataWithPath() throws IOException { + initializeTestEnvironment(); + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData; + + // Test with valid path + Path hFilePath = hStoreFiles.get(2).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + + // Test with another valid path + hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Test with a filename where corresponding HStoreFile in not present + hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName"); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("Store file corresponding to " + hFilePath + " doesn't exist")); + } + + @Test + public void testPrefetchWhenDataTieringEnabled() throws IOException { + setPrefetchBlocksOnOpen(); + this.blockCache = initializeTestEnvironment(); + // Evict blocks from cache by closing the files and passing evict on close. + // Then initialize the reader again. Since Prefetch on open is set to true, it should prefetch + // those blocks. + for (HStoreFile file : hStoreFiles) { + file.closeStoreFile(true); + file.initReader(); + } + + // Since we have one cold file among four files, only three should get prefetched. + Optional>> fullyCachedFiles = blockCache.getFullyCachedFiles(); + assertTrue("We should get the fully cached files from the cache", fullyCachedFiles.isPresent()); + Waiter.waitFor(defaultConf, 10000, () -> fullyCachedFiles.get().size() == 3); + assertEquals("Number of fully cached files are incorrect", 3, fullyCachedFiles.get().size()); + } + + private void setPrefetchBlocksOnOpen() { + defaultConf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); + } + + @Test + public void testColdDataFiles() throws IOException { + initializeTestEnvironment(); + Set allCachedBlocks = new HashSet<>(); + for (HStoreFile file : hStoreFiles) { + allCachedBlocks.add(new BlockCacheKey(file.getPath(), 0, true, BlockType.DATA)); + } + + // Verify hStoreFile3 is identified as cold data + DataTieringMethodCallerWithPath methodCallerWithPath = DataTieringManager::isHotData; + Path hFilePath = hStoreFiles.get(3).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, false); + + // Verify all the other files in hStoreFiles are hot data + for (int i = 0; i < hStoreFiles.size() - 1; i++) { + hFilePath = hStoreFiles.get(i).getPath(); + testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + } + + try { + Set coldFilePaths = dataTieringManager.getColdDataFiles(allCachedBlocks); + assertEquals(1, coldFilePaths.size()); + } catch (DataTieringException e) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + } + + @Test + public void testCacheCompactedBlocksOnWriteDataTieringDisabled() throws IOException { + setCacheCompactBlocksOnWrite(); + this.blockCache = initializeTestEnvironment(); + HRegion region = createHRegion("table3", this.blockCache); + testCacheCompactedBlocksOnWrite(region, true); + } + + @Test + public void testCacheCompactedBlocksOnWriteWithHotData() throws IOException { + setCacheCompactBlocksOnWrite(); + this.blockCache = initializeTestEnvironment(); + HRegion region = + createHRegion("table3", getConfWithCustomCellDataTieringEnabled(5 * DAY), this.blockCache); + testCacheCompactedBlocksOnWrite(region, true); + } + + @Test + public void testCacheCompactedBlocksOnWriteWithColdData() throws IOException { + setCacheCompactBlocksOnWrite(); + this.blockCache = initializeTestEnvironment(); + HRegion region = + createHRegion("table3", getConfWithCustomCellDataTieringEnabled(DAY), this.blockCache); + testCacheCompactedBlocksOnWrite(region, false); + } + + private void setCacheCompactBlocksOnWrite() { + defaultConf.setBoolean(CacheConfig.CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, true); + } + + private void testCacheCompactedBlocksOnWrite(HRegion region, boolean expectDataBlocksCached) + throws IOException { + HStore hStore = createHStore(region, "cf1"); + createTestFilesForCompaction(hStore); + hStore.refreshStoreFiles(); + + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + + long initialStoreFilesCount = hStore.getStorefilesCount(); + long initialCacheDataBlockCount = blockCache.getDataBlockCount(); + assertEquals(3, initialStoreFilesCount); + assertEquals(0, initialCacheDataBlockCount); + + region.compact(true); + + long compactedStoreFilesCount = hStore.getStorefilesCount(); + long compactedCacheDataBlockCount = blockCache.getDataBlockCount(); + assertEquals(1, compactedStoreFilesCount); + assertEquals(expectDataBlocksCached, compactedCacheDataBlockCount > 0); + } + + private void createTestFilesForCompaction(HStore hStore) throws IOException { + long currentTime = System.currentTimeMillis(); + Path storeDir = hStore.getStoreContext().getFamilyStoreDirectoryPath(); + Configuration configuration = hStore.getReadOnlyConfiguration(); + + HRegionFileSystem regionFS = hStore.getHRegion().getRegionFileSystem(); + + createHStoreFile(storeDir, configuration, currentTime - 2 * DAY, regionFS); + createHStoreFile(storeDir, configuration, currentTime - 3 * DAY, regionFS); + createHStoreFile(storeDir, configuration, currentTime - 4 * DAY, regionFS); + } + + @Test + public void testPickColdDataFiles() throws IOException { + initializeTestEnvironment(); + Map coldDataFiles = dataTieringManager.getColdFilesList(); + assertEquals(1, coldDataFiles.size()); + // hStoreFiles[3] is the cold file. + assert (coldDataFiles.containsKey(hStoreFiles.get(3).getFileInfo().getActiveFileName())); + } + + /* + * Verify that two cold blocks(both) are evicted when bucket reaches its capacity. The hot file + * remains in the cache. + */ + @Test + public void testBlockEvictions() throws Exception { + initializeTestEnvironment(); + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with cold data files and a block with hot data. + // hStoreFiles.get(3) is a cold data file, while hStoreFiles.get(0) is a hot file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block into cache with hot data which should trigger the eviction + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 2 hot blocks blocks only. + // Both cold blocks of 8KB will be evicted to make room for 1 block of 8KB + an additional + // space. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0); + } + + /* + * Verify that two cold blocks(both) are evicted when bucket reaches its capacity, but one cold + * block remains in the cache since the required space is freed. + */ + @Test + public void testBlockEvictionsAllColdBlocks() throws Exception { + initializeTestEnvironment(); + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with three cold data blocks. + // hStoreFiles.get(3) is a cold data file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 16384, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block into cache with hot data which should trigger the eviction + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 1 cold block and a newly added hot block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 1, 1); + } + + /* + * Verify that a hot block evicted along with a cold block when bucket reaches its capacity. + */ + @Test + public void testBlockEvictionsHotBlocks() throws Exception { + initializeTestEnvironment(); + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with two hot data blocks and one cold data block + // hStoreFiles.get(0) is a hot data file and hStoreFiles.get(3) is a cold data file. + Set cacheKeys = new HashSet<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional block which should evict the only cold block with an additional hot block. + BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket cache now contains 2 hot blocks. + // Only one of the older hot blocks is retained and other one is the newly added hot block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0); + } + + @Test + public void testFeatureKeyDisabled() throws Exception { + DataTieringManager.resetForTestingOnly(); + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, false); + initializeTestEnvironment(); + + try { + assertFalse(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); + // Verify that the DataaTieringManager instance is not instantiated in the + // instantiate call above. + assertNull(DataTieringManager.getInstance()); + + // Also validate that data temperature is not honoured. + long capacitySize = 40 * 1024; + int writeThreads = 3; + int writerQLen = 64; + int[] bucketSizes = new int[] { 8 * 1024 + 1024 }; + + // Setup: Create a bucket cache with lower capacity + BucketCache bucketCache = + new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, 8192, bucketSizes, + writeThreads, writerQLen, null, DEFAULT_ERROR_TOLERATION_DURATION, defaultConf); + + // Create three Cache keys with two hot data blocks and one cold data block + // hStoreFiles.get(0) is a hot data file and hStoreFiles.get(3) is a cold data file. + List cacheKeys = new ArrayList<>(); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 8192, true, BlockType.DATA)); + cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true, BlockType.DATA)); + + // Create dummy data to be cached and fill the cache completely. + CacheTestUtils.HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(8192, 3); + + int blocksIter = 0; + for (BlockCacheKey key : cacheKeys) { + LOG.info("Adding {}", key); + bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock()); + // Ensure that the block is persisted to the file. + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(key))); + } + + // Verify that the bucket cache contains 3 blocks. + assertEquals(3, bucketCache.getBackingMap().keySet().size()); + + // Add an additional hot block, which triggers eviction. + BlockCacheKey newKey = + new BlockCacheKey(hStoreFiles.get(2).getPath(), 0, true, BlockType.DATA); + CacheTestUtils.HFileBlockPair[] newBlock = CacheTestUtils.generateHFileBlocks(8192, 1); + + bucketCache.cacheBlock(newKey, newBlock[0].getBlock()); + Waiter.waitFor(defaultConf, 10000, 100, + () -> (bucketCache.getBackingMap().containsKey(newKey))); + + // Verify that the bucket still contains the only cold block and one newly added hot block. + // The older hot blocks are evicted and data-tiering mechanism does not kick in to evict + // the cold block. + validateBlocks(bucketCache.getBackingMap().keySet(), 2, 1, 1); + } finally { + DataTieringManager.resetForTestingOnly(); + defaultConf.setBoolean(DataTieringManager.GLOBAL_DATA_TIERING_ENABLED_KEY, true); + assertTrue(DataTieringManager.instantiate(defaultConf, testOnlineRegions)); + } + } + + @Test + public void testCacheConfigShouldCacheFile() throws Exception { + initializeTestEnvironment(); + // Verify that the API shouldCacheFileBlock returns the result correctly. + // hStoreFiles[0], hStoreFiles[1], hStoreFiles[2] are hot files. + // hStoreFiles[3] is a cold file. + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(0).getFileInfo().getHFileInfo(), hStoreFiles.get(0).getFileInfo().getConf())); + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(1).getFileInfo().getHFileInfo(), hStoreFiles.get(1).getFileInfo().getConf())); + assertTrue(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(2).getFileInfo().getHFileInfo(), hStoreFiles.get(2).getFileInfo().getConf())); + assertFalse(cacheConf.shouldCacheBlockOnRead(BlockCategory.DATA, + hStoreFiles.get(3).getFileInfo().getHFileInfo(), hStoreFiles.get(3).getFileInfo().getConf())); + } + + @Test + public void testCacheOnReadColdFile() throws Exception { + this.blockCache = initializeTestEnvironment(); + // hStoreFiles[3] is a cold file. the blocks should not get loaded after a readBlock call. + HStoreFile hStoreFile = hStoreFiles.get(3); + BlockCacheKey cacheKey = new BlockCacheKey(hStoreFile.getPath(), 0, true, BlockType.DATA); + testCacheOnRead(hStoreFile, cacheKey, -1, false); + } + + @Test + public void testCacheOnReadHotFile() throws Exception { + this.blockCache = initializeTestEnvironment(); + // hStoreFiles[0] is a hot file. the blocks should get loaded after a readBlock call. + HStoreFile hStoreFile = hStoreFiles.get(0); + BlockCacheKey cacheKey = + new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); + testCacheOnRead(hStoreFile, cacheKey, -1, true); + } + + private void testCacheOnRead(HStoreFile hStoreFile, BlockCacheKey key, long onDiskBlockSize, + boolean expectedCached) throws Exception { + // Execute the read block API which will try to cache the block if the block is a hot block. + hStoreFile.getReader().getHFileReader().readBlock(key.getOffset(), onDiskBlockSize, true, false, + false, false, key.getBlockType(), DataBlockEncoding.NONE); + // Validate that the hot block gets cached and cold block is not cached. + HFileBlock block = (HFileBlock) blockCache.getBlock(key, false, false, false); + if (expectedCached) { + assertNotNull(block); + } else { + assertNull(block); + } + } + + private void validateBlocks(Set keys, int expectedTotalKeys, int expectedHotBlocks, + int expectedColdBlocks) { + int numHotBlocks = 0, numColdBlocks = 0; + + Waiter.waitFor(defaultConf, 10000, 100, () -> (expectedTotalKeys == keys.size())); + int iter = 0; + for (BlockCacheKey key : keys) { + try { + if (dataTieringManager.isHotData(key)) { + numHotBlocks++; + } else { + numColdBlocks++; + } + } catch (Exception e) { + LOG.debug("Error validating priority for key {}", key, e); + fail(e.getMessage()); + } + } + assertEquals(expectedHotBlocks, numHotBlocks); + assertEquals(expectedColdBlocks, numColdBlocks); + } + + private void testDataTieringMethodWithPath(DataTieringMethodCallerWithPath caller, Path path, + boolean expectedResult, DataTieringException exception) { + try { + boolean value = caller.call(dataTieringManager, path); + if (exception != null) { + fail("Expected DataTieringException to be thrown"); + } + assertEquals(expectedResult, value); + } catch (DataTieringException e) { + if (exception == null) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + assertEquals(exception.getMessage(), e.getMessage()); + } + } + + private void testDataTieringMethodWithKey(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, boolean expectedResult, DataTieringException exception) { + try { + boolean value = caller.call(dataTieringManager, key); + if (exception != null) { + fail("Expected DataTieringException to be thrown"); + } + assertEquals(expectedResult, value); + } catch (DataTieringException e) { + if (exception == null) { + fail("Unexpected DataTieringException: " + e.getMessage()); + } + assertEquals(exception.getMessage(), e.getMessage()); + } + } + + private void testDataTieringMethodWithPathExpectingException( + DataTieringMethodCallerWithPath caller, Path path, DataTieringException exception) { + testDataTieringMethodWithPath(caller, path, false, exception); + } + + private void testDataTieringMethodWithPathNoException(DataTieringMethodCallerWithPath caller, + Path path, boolean expectedResult) { + testDataTieringMethodWithPath(caller, path, expectedResult, null); + } + + private void testDataTieringMethodWithKeyExpectingException(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, DataTieringException exception) { + testDataTieringMethodWithKey(caller, key, false, exception); + } + + private void testDataTieringMethodWithKeyNoException(DataTieringMethodCallerWithKey caller, + BlockCacheKey key, boolean expectedResult) { + testDataTieringMethodWithKey(caller, key, expectedResult, null); + } + + private static BlockCache initializeTestEnvironment() throws IOException { + BlockCache blockCache = setupFileSystemAndCache(); + setupOnlineRegions(blockCache); + return blockCache; + } + + private static BlockCache setupFileSystemAndCache() throws IOException { + fs = HFileSystem.get(defaultConf); + BlockCache blockCache = BlockCacheFactory.createBlockCache(defaultConf); + cacheConf = new CacheConfig(defaultConf, blockCache); + return blockCache; + } + + private static void setupOnlineRegions(BlockCache blockCache) throws IOException { + testOnlineRegions.clear(); + hStoreFiles.clear(); + long day = 24 * 60 * 60 * 1000; + long currentTime = System.currentTimeMillis(); + + HRegion region1 = createHRegion("table1", blockCache); + + HStore hStore11 = createHStore(region1, "cf1", getConfWithCustomCellDataTieringEnabled(day)); + hStoreFiles.add(createHStoreFile(hStore11.getStoreContext().getFamilyStoreDirectoryPath(), + hStore11.getReadOnlyConfiguration(), currentTime, region1.getRegionFileSystem())); + hStore11.refreshStoreFiles(); + HStore hStore12 = createHStore(region1, "cf2"); + hStoreFiles.add(createHStoreFile(hStore12.getStoreContext().getFamilyStoreDirectoryPath(), + hStore12.getReadOnlyConfiguration(), currentTime - day, region1.getRegionFileSystem())); + hStore12.refreshStoreFiles(); + + region1.stores.put(Bytes.toBytes("cf1"), hStore11); + region1.stores.put(Bytes.toBytes("cf2"), hStore12); + + HRegion region2 = createHRegion("table2", + getConfWithCustomCellDataTieringEnabled((long) (2.5 * day)), blockCache); + + HStore hStore21 = createHStore(region2, "cf1"); + hStoreFiles.add(createHStoreFile(hStore21.getStoreContext().getFamilyStoreDirectoryPath(), + hStore21.getReadOnlyConfiguration(), currentTime - 2 * day, region2.getRegionFileSystem())); + hStore21.refreshStoreFiles(); + HStore hStore22 = createHStore(region2, "cf2"); + hStoreFiles.add(createHStoreFile(hStore22.getStoreContext().getFamilyStoreDirectoryPath(), + hStore22.getReadOnlyConfiguration(), currentTime - 3 * day, region2.getRegionFileSystem())); + hStore22.refreshStoreFiles(); + + region2.stores.put(Bytes.toBytes("cf1"), hStore21); + region2.stores.put(Bytes.toBytes("cf2"), hStore22); + + for (HStoreFile file : hStoreFiles) { + file.initReader(); + } + + testOnlineRegions.put(region1.getRegionInfo().getEncodedName(), region1); + testOnlineRegions.put(region2.getRegionInfo().getEncodedName(), region2); + } + + private static HRegion createHRegion(String table, BlockCache blockCache) throws IOException { + return createHRegion(table, defaultConf, blockCache); + } + + private static HRegion createHRegion(String table, Configuration conf, BlockCache blockCache) + throws IOException { + TableName tableName = TableName.valueOf(table); + + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) + .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, + conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .build(); + RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); + + Configuration testConf = new Configuration(conf); + CommonFSUtils.setRootDir(testConf, testDir); + HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); + + HRegion region = new HRegion(regionFs, null, conf, htd, null); + // Manually sets the BlockCache for the HRegion instance. + // This is necessary because the region server is not started within this method, + // and therefore the BlockCache needs to be explicitly configured. + region.setBlockCache(blockCache); + return region; + } + + private static HStore createHStore(HRegion region, String columnFamily) throws IOException { + return createHStore(region, columnFamily, defaultConf); + } + + private static HStore createHStore(HRegion region, String columnFamily, Configuration conf) + throws IOException { + ColumnFamilyDescriptor columnFamilyDescriptor = + ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)) + .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) + .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, + conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .build(); + + return new HStore(region, columnFamilyDescriptor, conf, false); + } + + private static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp, + HRegionFileSystem regionFs) throws IOException { + String columnFamily = storeDir.getName(); + + StoreFileWriter storeFileWriter = new StoreFileWriter.Builder(conf, cacheConf, fs) + .withOutputDir(storeDir).withFileContext(new HFileContextBuilder().build()).build(); + + writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); + + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true); + } + + private static Configuration getConfWithCustomCellDataTieringEnabled(long hotDataAge) { + Configuration conf = new Configuration(defaultConf); + conf.set(DataTieringManager.DATATIERING_KEY, DataTieringType.CUSTOM.name()); + conf.set(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(hotDataAge)); + return conf; + } + + /** + * Writes random data to a store file with rows arranged in lexicographically increasing order. + * Each row is generated using the {@link #nextString()} method, ensuring that each subsequent row + * is lexicographically larger than the previous one. + */ + private static void writeStoreFileRandomData(final StoreFileWriter writer, byte[] columnFamily, + long timestamp) throws IOException { + int cellsPerFile = 10; + byte[] qualifier = Bytes.toBytes("qualifier"); + byte[] value = generateRandomBytes(4 * 1024); + try { + for (int i = 0; i < cellsPerFile; i++) { + byte[] row = Bytes.toBytes(nextString()); + writer.append(new KeyValue(row, columnFamily, qualifier, timestamp, value)); + } + } finally { + writer.appendTrackedTimestampsToMetadata(); + TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); + timeRangeTracker.setMin(timestamp); + timeRangeTracker.setMax(timestamp); + writer.appendCustomCellTimestampsToMetadata(timeRangeTracker); + writer.close(); + } + } + + private static byte[] generateRandomBytes(int sizeInBytes) { + Random random = new Random(); + byte[] randomBytes = new byte[sizeInBytes]; + random.nextBytes(randomBytes); + return randomBytes; + } + + /** + * Returns the lexicographically larger string every time it's called. + */ + private static String nextString() { + if (rowKeyString == null || rowKeyString.isEmpty()) { + rowKeyString = "a"; + } + char lastChar = rowKeyString.charAt(rowKeyString.length() - 1); + if (lastChar < 'z') { + rowKeyString = rowKeyString.substring(0, rowKeyString.length() - 1) + (char) (lastChar + 1); + } else { + rowKeyString = rowKeyString + "a"; + } + return rowKeyString; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java new file mode 100644 index 000000000000..b2c363bf53ac --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.UUID; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy; +import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestCustomCellTieredCompactionPolicy { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCustomCellTieredCompactionPolicy.class); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + public static final byte[] FAMILY = Bytes.toBytes("cf"); + + private HStoreFile createFile(Path file, long minValue, long maxValue, long size, int seqId) + throws IOException { + return createFile(mockRegionInfo(), file, minValue, maxValue, size, seqId, 0); + } + + private HStoreFile createFile(RegionInfo regionInfo, Path file, long minValue, long maxValue, + long size, int seqId, long ageInDisk) throws IOException { + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + HRegionFileSystem regionFileSystem = + new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, file, regionInfo); + MockHStoreFile msf = new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId); + TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); + timeRangeTracker.setMin(minValue); + timeRangeTracker.setMax(maxValue); + msf.setMetadataValue(CUSTOM_TIERING_TIME_RANGE, TimeRangeTracker.toByteArray(timeRangeTracker)); + return msf; + } + + private CustomDateTieredCompactionPolicy mockAndCreatePolicy() throws Exception { + RegionInfo mockedRegionInfo = mockRegionInfo(); + return mockAndCreatePolicy(mockedRegionInfo); + } + + private CustomDateTieredCompactionPolicy mockAndCreatePolicy(RegionInfo regionInfo) + throws Exception { + StoreConfigInformation mockedStoreConfig = mock(StoreConfigInformation.class); + when(mockedStoreConfig.getRegionInfo()).thenReturn(regionInfo); + CustomDateTieredCompactionPolicy policy = + new CustomDateTieredCompactionPolicy(TEST_UTIL.getConfiguration(), mockedStoreConfig); + return policy; + } + + private RegionInfo mockRegionInfo() { + RegionInfo mockedRegionInfo = mock(RegionInfo.class); + when(mockedRegionInfo.getEncodedName()).thenReturn("1234567890987654321"); + return mockedRegionInfo; + } + + private Path preparePath() throws Exception { + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + Path file = + new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString().replaceAll("-", "")); + fs.create(file); + return file; + } + + @Test + public void testGetCompactBoundariesForMajorNoOld() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + assertEquals(1, + ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size()); + } + + @Test + public void testGetCompactBoundariesForMajorAllOld() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + // The default cut off age is 10 years, so any of the min/max value there should get in the old + // tier + files.add(createFile(file, 0, 1, 1024, 0)); + files.add(createFile(file, 2, 3, 1024, 1)); + assertEquals(2, + ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size()); + } + + @Test + public void testGetCompactBoundariesForMajorOneOnEachSide() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, 0, 1, 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + assertEquals(3, + ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size()); + } + + @Test + public void testGetCompactBoundariesForMajorOneCrossing() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, 0, EnvironmentEdgeManager.currentTime(), 1024, 0)); + assertEquals(3, + ((DateTieredCompactionRequest) policy.selectMajorCompaction(files)).getBoundaries().size()); + } + + @FunctionalInterface + interface PolicyValidator { + void accept(T t, U u) throws Exception; + } + + private void testShouldPerformMajorCompaction(long min, long max, int numFiles, + PolicyValidator> validation) + throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + RegionInfo mockedRegionInfo = mockRegionInfo(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(timeMachine); + for (int i = 0; i < numFiles; i++) { + MockHStoreFile mockedSFile = (MockHStoreFile) createFile(mockedRegionInfo, file, min, max, + 1024, 0, HConstants.DEFAULT_MAJOR_COMPACTION_PERIOD); + mockedSFile.setIsMajor(true); + files.add(mockedSFile); + } + EnvironmentEdgeManager.reset(); + validation.accept(policy, files); + } + + @Test + public void testShouldPerformMajorCompactionOneFileCrossing() throws Exception { + long max = EnvironmentEdgeManager.currentTime(); + testShouldPerformMajorCompaction(0, max, 1, + (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f))); + } + + @Test + public void testShouldPerformMajorCompactionOneFileMinMaxLow() throws Exception { + testShouldPerformMajorCompaction(0, 1, 1, + (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f))); + } + + @Test + public void testShouldPerformMajorCompactionOneFileMinMaxHigh() throws Exception { + long currentTime = EnvironmentEdgeManager.currentTime(); + testShouldPerformMajorCompaction(currentTime, currentTime, 1, + (p, f) -> assertFalse(p.shouldPerformMajorCompaction(f))); + } + + @Test + public void testShouldPerformMajorCompactionTwoFilesMinMaxHigh() throws Exception { + long currentTime = EnvironmentEdgeManager.currentTime(); + testShouldPerformMajorCompaction(currentTime, currentTime, 2, + (p, f) -> assertTrue(p.shouldPerformMajorCompaction(f))); + } + + @Test + public void testSelectMinorCompactionTwoFilesNoOld() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + // Shouldn't do minor compaction, as minimum number of files + // for minor compactions is 3 + assertEquals(0, policy.selectMinorCompaction(files, true, true).getFiles().size()); + } + + @Test + public void testSelectMinorCompactionThreeFilesNoOld() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 2)); + assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size()); + } + + @Test + public void testSelectMinorCompactionThreeFilesAllOld() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, 0, 1, 1024, 0)); + files.add(createFile(file, 1, 2, 1024, 1)); + files.add(createFile(file, 3, 4, 1024, 2)); + assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size()); + } + + @Test + public void testSelectMinorCompactionThreeFilesOneOldTwoNew() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, 0, 1, 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 2)); + assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size()); + } + + @Test + public void testSelectMinorCompactionThreeFilesTwoOldOneNew() throws Exception { + CustomDateTieredCompactionPolicy policy = mockAndCreatePolicy(); + Path file = preparePath(); + ArrayList files = new ArrayList<>(); + files.add(createFile(file, 0, 1, 1024, 0)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 1)); + files.add(createFile(file, EnvironmentEdgeManager.currentTime(), + EnvironmentEdgeManager.currentTime(), 1024, 2)); + assertEquals(3, policy.selectMinorCompaction(files, true, true).getFiles().size()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index b24c1f2ed84a..585482c94093 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -224,7 +225,8 @@ public void testHotDataWithPath() throws IOException { // Test with a filename where corresponding HStoreFile in not present hFilePath = new Path(hStoreFiles.get(0).getPath().getParent(), "incorrectFileName"); - testDataTieringMethodWithPathNoException(methodCallerWithPath, hFilePath, true); + testDataTieringMethodWithPathExpectingException(methodCallerWithPath, hFilePath, + new DataTieringException("Store file corresponding to " + hFilePath + " doesn't exist")); } @Test @@ -597,19 +599,21 @@ public void testCacheConfigShouldCacheFile() throws Exception { @Test public void testCacheOnReadColdFile() throws Exception { + initializeTestEnvironment(); // hStoreFiles[3] is a cold file. the blocks should not get loaded after a readBlock call. HStoreFile hStoreFile = hStoreFiles.get(3); BlockCacheKey cacheKey = new BlockCacheKey(hStoreFile.getPath(), 0, true, BlockType.DATA); - testCacheOnRead(hStoreFile, cacheKey, 23025, false); + testCacheOnRead(hStoreFile, cacheKey, -1, false); } @Test public void testCacheOnReadHotFile() throws Exception { + initializeTestEnvironment(); // hStoreFiles[0] is a hot file. the blocks should get loaded after a readBlock call. HStoreFile hStoreFile = hStoreFiles.get(0); BlockCacheKey cacheKey = new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true, BlockType.DATA); - testCacheOnRead(hStoreFile, cacheKey, 23025, true); + testCacheOnRead(hStoreFile, cacheKey, -1, true); } private void testCacheOnRead(HStoreFile hStoreFile, BlockCacheKey key, long onDiskBlockSize, @@ -618,7 +622,7 @@ private void testCacheOnRead(HStoreFile hStoreFile, BlockCacheKey key, long onDi hStoreFile.getReader().getHFileReader().readBlock(key.getOffset(), onDiskBlockSize, true, false, false, false, key.getBlockType(), DataBlockEncoding.NONE); // Validate that the hot block gets cached and cold block is not cached. - HFileBlock block = (HFileBlock) blockCache.getBlock(key, false, false, false, BlockType.DATA); + HFileBlock block = (HFileBlock) blockCache.getBlock(key, false, false, false); if (expectedCached) { assertNotNull(block); } else { @@ -803,7 +807,6 @@ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotData return conf; } - static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp, HRegionFileSystem regionFs) throws IOException { String columnFamily = storeDir.getName(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCustomCellTieredCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCustomCellTieredCompactor.java new file mode 100644 index 000000000000..331dd41e4f1b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCustomCellTieredCompactor.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.compactions; + +import static org.apache.hadoop.hbase.regionserver.CustomTieringMultiFileWriter.CUSTOM_TIERING_TIME_RANGE; +import static org.apache.hadoop.hbase.regionserver.compactions.CustomCellTieringValueProvider.TIERING_CELL_QUALIFIER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestCustomCellTieredCompactor { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCustomCellTieredCompactor.class); + + public static final byte[] FAMILY = Bytes.toBytes("cf"); + + protected HBaseTestingUtility utility; + + protected Admin admin; + + @Before + public void setUp() throws Exception { + utility = new HBaseTestingUtility(); + utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10); + utility.startMiniCluster(); + } + + @After + public void tearDown() throws Exception { + utility.shutdownMiniCluster(); + } + + @Test + public void testCustomCellTieredCompactor() throws Exception { + ColumnFamilyDescriptorBuilder clmBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + clmBuilder.setValue("hbase.hstore.engine.class", CustomTieredStoreEngine.class.getName()); + clmBuilder.setValue(TIERING_CELL_QUALIFIER, "date"); + TableName tableName = TableName.valueOf("testCustomCellTieredCompactor"); + TableDescriptorBuilder tblBuilder = TableDescriptorBuilder.newBuilder(tableName); + tblBuilder.setColumnFamily(clmBuilder.build()); + utility.getAdmin().createTable(tblBuilder.build()); + utility.waitTableAvailable(tableName); + Connection connection = utility.getConnection(); + Table table = connection.getTable(tableName); + long recordTime = System.currentTimeMillis(); + // write data and flush multiple store files: + for (int i = 0; i < 6; i++) { + List puts = new ArrayList<>(2); + Put put = new Put(Bytes.toBytes(i)); + put.addColumn(FAMILY, Bytes.toBytes("val"), Bytes.toBytes("v" + i)); + put.addColumn(FAMILY, Bytes.toBytes("date"), + Bytes.toBytes(recordTime - (11L * 366L * 24L * 60L * 60L * 1000L))); + puts.add(put); + put = new Put(Bytes.toBytes(i + 1000)); + put.addColumn(FAMILY, Bytes.toBytes("val"), Bytes.toBytes("v" + (i + 1000))); + put.addColumn(FAMILY, Bytes.toBytes("date"), Bytes.toBytes(recordTime)); + puts.add(put); + table.put(puts); + utility.flush(tableName); + } + table.close(); + long firstCompactionTime = System.currentTimeMillis(); + utility.getAdmin().majorCompact(tableName); + Waiter.waitFor(utility.getConfiguration(), 5000, + () -> utility.getMiniHBaseCluster().getMaster().getLastMajorCompactionTimestamp(tableName) + > firstCompactionTime); + long numHFiles = utility.getNumHFiles(tableName, FAMILY); + // The first major compaction would have no means to detect more than one tier, + // because without the min/max values available in the file info portion of the selected files + // for compaction, CustomCellDateTieredCompactionPolicy has no means + // to calculate the proper boundaries. + assertEquals(1, numHFiles); + utility.getMiniHBaseCluster().getRegions(tableName).get(0).getStore(FAMILY).getStorefiles() + .forEach(file -> { + byte[] rangeBytes = file.getMetadataValue(CUSTOM_TIERING_TIME_RANGE); + assertNotNull(rangeBytes); + try { + TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(rangeBytes); + assertEquals((recordTime - (11L * 366L * 24L * 60L * 60L * 1000L)), + timeRangeTracker.getMin()); + assertEquals(recordTime, timeRangeTracker.getMax()); + } catch (IOException e) { + fail(e.getMessage()); + } + }); + // now do major compaction again, to make sure we write two separate files + long secondCompactionTime = System.currentTimeMillis(); + utility.getAdmin().majorCompact(tableName); + Waiter.waitFor(utility.getConfiguration(), 5000, + () -> utility.getMiniHBaseCluster().getMaster().getLastMajorCompactionTimestamp(tableName) + > secondCompactionTime); + numHFiles = utility.getNumHFiles(tableName, FAMILY); + assertEquals(2, numHFiles); + utility.getMiniHBaseCluster().getRegions(tableName).get(0).getStore(FAMILY).getStorefiles() + .forEach(file -> { + byte[] rangeBytes = file.getMetadataValue(CUSTOM_TIERING_TIME_RANGE); + assertNotNull(rangeBytes); + try { + TimeRangeTracker timeRangeTracker = TimeRangeTracker.parseFrom(rangeBytes); + assertEquals(timeRangeTracker.getMin(), timeRangeTracker.getMax()); + } catch (IOException e) { + fail(e.getMessage()); + } + }); + } +} From a18ce11557d85695c71186cf5f2e8ed6dd9e56ca Mon Sep 17 00:00:00 2001 From: Huginn <63332600+Huginn-kio@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:50:29 +0800 Subject: [PATCH 048/336] HBASE-29570 Set no watches on the node when recursively deleting the node and its child nodes (#7271) Signed-off-by: Duo Zhang (cherry picked from commit 64a192197c313a43f008c692bbe935f347c77dba) --- .../src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 334c12b532fd..8d22747b70fa 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -966,7 +966,7 @@ public static void deleteNodeRecursivelyMultiOrSequential(ZKWatcher zkw, ops.add(ZKUtilOp.deleteNodeFailSilent(children.get(i))); } try { - if (zkw.getRecoverableZooKeeper().exists(eachRoot, zkw) != null) { + if (zkw.getRecoverableZooKeeper().exists(eachRoot, null) != null) { ops.add(ZKUtilOp.deleteNodeFailSilent(eachRoot)); } } catch (InterruptedException e) { From a22473aeb2aab2e50be2e9d3e64d978e3d790f65 Mon Sep 17 00:00:00 2001 From: Hari Krishna Dara Date: Fri, 5 Sep 2025 09:34:15 +0530 Subject: [PATCH 049/336] HBASE-29558: Fix for TestShellNoCluster along with code refactoring and cleanup (#7256) (#7274) Signed-off-by: Nihal Jain --- .../hbase/client/AbstractTestShell.java | 115 ++++++------------ .../hadoop/hbase/client/RubyShellTest.java | 107 ++++++++++++++++ .../hadoop/hbase/client/TestAdminShell.java | 16 +-- .../hadoop/hbase/client/TestAdminShell2.java | 37 ------ .../hbase/client/TestChangeSftShell.java | 46 ------- .../hbase/client/TestListTablesShell.java | 12 +- .../hadoop/hbase/client/TestQuotasShell.java | 3 +- .../hadoop/hbase/client/TestRSGroupShell.java | 14 ++- .../hbase/client/TestReplicationShell.java | 3 +- .../apache/hadoop/hbase/client/TestShell.java | 6 +- .../hbase/client/TestShellNoCluster.java | 41 +++---- .../hadoop/hbase/client/TestTableShell.java | 3 +- ...{admin2_test.rb => admin2_test_cluster.rb} | 0 ...test.rb => balancer_utils_test_cluster.rb} | 0 ...uster.rb => connection_test_no_cluster.rb} | 0 .../{hbase_test.rb => hbase_test_cluster.rb} | 0 ...test.rb => security_admin_test_cluster.rb} | 0 ...or_test.rb => taskmonitor_test_cluster.rb} | 0 ...> visibility_labels_admin_test_cluster.rb} | 0 .../src/test/ruby/no_cluster_tests_runner.rb | 94 -------------- ...mands_test.rb => commands_test_cluster.rb} | 0 ...rter_test.rb => converter_test_cluster.rb} | 0 ...tter_test.rb => formatter_test_cluster.rb} | 0 ...{shell_test.rb => general_test_cluster.rb} | 0 ...cks_test.rb => list_locks_test_cluster.rb} | 0 ...est.rb => list_procedures_test_cluster.rb} | 0 ...test.rb => noninteractive_test_cluster.rb} | 0 ...hell_test.rb => sftchange_test_cluster.rb} | 0 hbase-shell/src/test/ruby/tests_runner.rb | 37 +----- 29 files changed, 186 insertions(+), 348 deletions(-) create mode 100644 hbase-shell/src/test/java/org/apache/hadoop/hbase/client/RubyShellTest.java delete mode 100644 hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java delete mode 100644 hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java rename hbase-shell/src/test/ruby/hbase/{admin2_test.rb => admin2_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{balancer_utils_test.rb => balancer_utils_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{test_connection_no_cluster.rb => connection_test_no_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{hbase_test.rb => hbase_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{security_admin_test.rb => security_admin_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{taskmonitor_test.rb => taskmonitor_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/hbase/{visibility_labels_admin_test.rb => visibility_labels_admin_test_cluster.rb} (100%) delete mode 100644 hbase-shell/src/test/ruby/no_cluster_tests_runner.rb rename hbase-shell/src/test/ruby/shell/{commands_test.rb => commands_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{converter_test.rb => converter_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{formatter_test.rb => formatter_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{shell_test.rb => general_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{list_locks_test.rb => list_locks_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{list_procedures_test.rb => list_procedures_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{noninteractive_test.rb => noninteractive_test_cluster.rb} (100%) rename hbase-shell/src/test/ruby/shell/{sftchange_shell_test.rb => sftchange_test_cluster.rb} (100%) diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java index 63f651ddf432..ecd1ea6c5974 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java @@ -18,103 +18,62 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; -import org.jruby.embed.PathType; +import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.jruby.embed.ScriptingContainer; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public abstract class AbstractTestShell { +public abstract class AbstractTestShell implements RubyShellTest { + protected final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + protected final ScriptingContainer jruby = new ScriptingContainer(); - private static final Logger LOG = LoggerFactory.getLogger(AbstractTestShell.class); + protected boolean erasureCodingSupported = false; - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - protected final static ScriptingContainer jruby = new ScriptingContainer(); - - protected static void setUpConfig() throws IOException { - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setBoolean("hbase.quota.enabled", true); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false); - conf.setInt("hfile.format.version", 3); - - // Below settings are necessary for task monitor test. - conf.setInt(HConstants.MASTER_INFO_PORT, 0); - conf.setInt(HConstants.REGIONSERVER_INFO_PORT, 0); - conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true); - // Security setup configuration - SecureTestUtil.enableSecurity(conf); - VisibilityTestUtil.enableVisiblityLabels(conf); + public HBaseTestingUtility getTEST_UTIL() { + return TEST_UTIL; } - protected static void setUpJRubyRuntime() { - setUpJRubyRuntime(Collections.emptyMap()); + public ScriptingContainer getJRuby() { + return jruby; } - protected static void setUpJRubyRuntime(Map extraVars) { - LOG.debug("Configure jruby runtime, cluster set to {}", TEST_UTIL); - List loadPaths = new ArrayList<>(2); - loadPaths.add("src/test/ruby"); - jruby.setLoadPaths(loadPaths); - jruby.put("$TEST_CLUSTER", TEST_UTIL); - for (Map.Entry entry : extraVars.entrySet()) { - jruby.put(entry.getKey(), entry.getValue()); - } - System.setProperty("jruby.jit.logging.verbose", "true"); - System.setProperty("jruby.jit.logging", "true"); - System.setProperty("jruby.native.verbose", "true"); + public String getSuitePattern() { + return "**/*_test.rb"; } - /** Returns comma separated list of ruby script names for tests */ - protected String getIncludeList() { - return ""; - } + @Before + public void setUp() throws Exception { + RubyShellTest.setUpConfig(this); + + // Start mini cluster + TEST_UTIL.startMiniCluster(1); + + RubyShellTest.setUpJRubyRuntime(this); - /** Returns comma separated list of ruby script names for tests to skip */ - protected String getExcludeList() { - return ""; + RubyShellTest.doTestSetup(this); } - @Test - public void testRunShellTests() throws IOException { - final String tests = getIncludeList(); - final String excludes = getExcludeList(); - if (!tests.isEmpty()) { - System.setProperty("shell.test.include", tests); + protected void setupDFS() throws IOException { + try { + ErasureCodingUtils.enablePolicy(FileSystem.get(TEST_UTIL.getConfiguration()), + "XOR-2-1-1024k"); + erasureCodingSupported = true; + } catch (UnsupportedOperationException e) { + LOG.info( + "Current hadoop version does not support erasure coding, only validation tests will run."); } - if (!excludes.isEmpty()) { - System.setProperty("shell.test.exclude", excludes); - } - LOG.info("Starting ruby tests. includes: {} excludes: {}", tests, excludes); - jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb"); } - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); - - // Start mini cluster - TEST_UTIL.startMiniCluster(1); - - setUpJRubyRuntime(); + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); } - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); + @Test + public void testRunShellTests() throws IOException { + RubyShellTest.testRunShellTests(this); } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/RubyShellTest.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/RubyShellTest.java new file mode 100644 index 000000000000..770acb66a9d5 --- /dev/null +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/RubyShellTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; +import org.jruby.embed.PathType; +import org.jruby.embed.ScriptingContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface RubyShellTest { + static Logger LOG = LoggerFactory.getLogger(RubyShellTest.class); + + HBaseTestingUtility getTEST_UTIL(); + + ScriptingContainer getJRuby(); + + /** Returns comma separated list of ruby script names for tests */ + default String getIncludeList() { + return ""; + } + + /** Returns comma separated list of ruby script names for tests to skip */ + default String getExcludeList() { + return ""; + } + + String getSuitePattern(); + + static void setUpConfig(RubyShellTest test) throws IOException { + Configuration conf = test.getTEST_UTIL().getConfiguration(); + conf.setInt("hbase.regionserver.msginterval", 100); + conf.setInt("hbase.client.pause", 250); + conf.setBoolean("hbase.quota.enabled", true); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false); + conf.setInt("hfile.format.version", 3); + + // Below settings are necessary for task monitor test. + conf.setInt(HConstants.MASTER_INFO_PORT, 0); + conf.setInt(HConstants.REGIONSERVER_INFO_PORT, 0); + conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true); + // Security setup configuration + SecureTestUtil.enableSecurity(conf); + VisibilityTestUtil.enableVisiblityLabels(conf); + } + + static void setUpJRubyRuntime(RubyShellTest test) { + setUpJRubyRuntime(test, Collections.emptyMap()); + } + + static void setUpJRubyRuntime(RubyShellTest test, Map extraVars) { + LOG.debug("Configure jruby runtime, cluster set to {}", test.getTEST_UTIL()); + List loadPaths = new ArrayList<>(2); + loadPaths.add("src/test/ruby"); + test.getJRuby().setLoadPaths(loadPaths); + test.getJRuby().put("$TEST_CLUSTER", test.getTEST_UTIL()); + for (Map.Entry entry : extraVars.entrySet()) { + test.getJRuby().put(entry.getKey(), entry.getValue()); + } + System.setProperty("jruby.jit.logging.verbose", "true"); + System.setProperty("jruby.jit.logging", "true"); + System.setProperty("jruby.native.verbose", "true"); + } + + static void doTestSetup(RubyShellTest test) { + System.setProperty("shell.test.suite_name", test.getClass().getSimpleName()); + System.setProperty("shell.test.suite_pattern", test.getSuitePattern()); + if (!test.getIncludeList().isEmpty()) { + System.setProperty("shell.test.include", test.getIncludeList()); + } + if (!test.getExcludeList().isEmpty()) { + System.setProperty("shell.test.exclude", test.getExcludeList()); + } + LOG.info("Starting ruby tests on script: {} includes: {} excludes: {}", + test.getClass().getSimpleName(), test.getIncludeList(), test.getExcludeList()); + } + + static void testRunShellTests(RubyShellTest test) throws IOException { + test.getJRuby().runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb"); + } +} diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java index 597ae903d1ce..2622c80ac642 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -38,15 +38,15 @@ public class TestAdminShell extends AbstractTestShell { HBaseClassTestRule.forClass(TestAdminShell.class); @Override - protected String getIncludeList() { + public String getIncludeList() { return "admin_test.rb"; } - protected static boolean erasureCodingSupported = false; + protected boolean erasureCodingSupported = false; - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); + @Before + public void setUp() throws Exception { + RubyShellTest.setUpConfig(this); // Start mini cluster // 3 datanodes needed for erasure coding checks @@ -61,7 +61,9 @@ public static void setUpBeforeClass() throws Exception { } // we'll use this extra variable to trigger some differences in the tests - setUpJRubyRuntime( + RubyShellTest.setUpJRubyRuntime(this, Collections.singletonMap("$ERASURE_CODING_SUPPORTED", erasureCodingSupported)); + + RubyShellTest.doTestSetup(this); } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java deleted file mode 100644 index 0bf67cdc8b45..000000000000 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; - -@Category({ ClientTests.class, LargeTests.class }) -public class TestAdminShell2 extends AbstractTestShell { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestAdminShell2.class); - - @Override - protected String getIncludeList() { - return "admin2_test.rb"; - } -} diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java deleted file mode 100644 index e8127afcc709..000000000000 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestChangeSftShell.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; - -@Category({ ClientTests.class, LargeTests.class }) -public class TestChangeSftShell extends AbstractTestShell { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestChangeSftShell.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); - - TEST_UTIL.startMiniCluster(3); - - setUpJRubyRuntime(); - } - - @Override - protected String getIncludeList() { - return "sftchange_shell_test.rb"; - } -} diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestListTablesShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestListTablesShell.java index b823b6fc3aa7..ffd80f91a381 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestListTablesShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestListTablesShell.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.experimental.categories.Category; @@ -30,17 +29,8 @@ public class TestListTablesShell extends AbstractTestShell { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestListTablesShell.class); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); - - TEST_UTIL.startMiniCluster(3); - - setUpJRubyRuntime(); - } - @Override - protected String getIncludeList() { + public String getIncludeList() { return "list_tables_test.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestQuotasShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestQuotasShell.java index d7b9fab42a3b..1550478d0a72 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestQuotasShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestQuotasShell.java @@ -25,13 +25,12 @@ @Category({ ClientTests.class, LargeTests.class }) public class TestQuotasShell extends AbstractTestShell { - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestQuotasShell.class); @Override - protected String getIncludeList() { + public String getIncludeList() { return "quotas_test.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java index b64f4a43ec87..3f77ca5386da 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.ClassRule; import org.junit.experimental.categories.Category; @@ -35,9 +35,9 @@ public class TestRSGroupShell extends AbstractTestShell { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestRSGroupShell.class); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpConfig(); + @Before + public void setUp() throws Exception { + RubyShellTest.setUpConfig(this); // enable rs group TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, @@ -48,11 +48,13 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); - setUpJRubyRuntime(); + RubyShellTest.setUpJRubyRuntime(this); + + RubyShellTest.doTestSetup(this); } @Override - protected String getIncludeList() { + public String getIncludeList() { return "rsgroup_shell_test.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java index ced1e7adda1d..5bf0b3a6328d 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java @@ -25,13 +25,12 @@ @Category({ ClientTests.class, LargeTests.class }) public class TestReplicationShell extends AbstractTestShell { - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestReplicationShell.class); @Override - protected String getIncludeList() { + public String getIncludeList() { return "replication_admin_test.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java index 47918f68019f..28b1fb59ef01 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java @@ -25,13 +25,11 @@ @Category({ ClientTests.class, LargeTests.class }) public class TestShell extends AbstractTestShell { - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestShell.class); @Override - protected String getExcludeList() { - return "replication_admin_test.rb,rsgroup_shell_test.rb,admin_test.rb,table_test.rb," - + "quotas_test.rb,admin2_test.rb,list_tables_test.rb"; + public String getSuitePattern() { + return "**/*_test_cluster.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java index c8685f34d197..5c312ec1dac0 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java @@ -17,51 +17,38 @@ */ package org.apache.hadoop.hbase.client; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.jruby.embed.PathType; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.ClassRule; -import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Category({ ClientTests.class, MediumTests.class }) public class TestShellNoCluster extends AbstractTestShell { - private static final Logger LOG = LoggerFactory.getLogger(TestShellNoCluster.class); - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestShellNoCluster.class); - @BeforeClass - public static void setUpBeforeClass() throws Exception { + @Before + public void setUp() throws Exception { + RubyShellTest.setUpConfig(this); + // no cluster - List loadPaths = new ArrayList<>(2); - loadPaths.add("src/test/ruby"); - jruby.setLoadPaths(loadPaths); - jruby.put("$TEST_CLUSTER", TEST_UTIL); - System.setProperty("jruby.jit.logging.verbose", "true"); - System.setProperty("jruby.jit.logging", "true"); - System.setProperty("jruby.native.verbose", "true"); + + RubyShellTest.setUpJRubyRuntime(this); + + RubyShellTest.doTestSetup(this); } - @AfterClass - public static void tearDownAfterClass() throws Exception { + @After + public void tearDown() throws Exception { // no cluster } - // Keep the same name so we override the with-a-cluster test @Override - @Test - public void testRunShellTests() throws IOException { - LOG.info("Start ruby tests without cluster"); - jruby.runScriptlet(PathType.CLASSPATH, "no_cluster_tests_runner.rb"); + public String getSuitePattern() { + return "**/*_test_no_cluster.rb"; } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java index 1ae56ac669bc..6158570cc058 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java @@ -25,13 +25,12 @@ @Category({ ClientTests.class, MediumTests.class }) public class TestTableShell extends AbstractTestShell { - @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestTableShell.class); @Override - protected String getIncludeList() { + public String getIncludeList() { return "table_test.rb"; } } diff --git a/hbase-shell/src/test/ruby/hbase/admin2_test.rb b/hbase-shell/src/test/ruby/hbase/admin2_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/admin2_test.rb rename to hbase-shell/src/test/ruby/hbase/admin2_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/balancer_utils_test.rb b/hbase-shell/src/test/ruby/hbase/balancer_utils_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/balancer_utils_test.rb rename to hbase-shell/src/test/ruby/hbase/balancer_utils_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb b/hbase-shell/src/test/ruby/hbase/connection_test_no_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb rename to hbase-shell/src/test/ruby/hbase/connection_test_no_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/hbase_test.rb b/hbase-shell/src/test/ruby/hbase/hbase_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/hbase_test.rb rename to hbase-shell/src/test/ruby/hbase/hbase_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/security_admin_test.rb b/hbase-shell/src/test/ruby/hbase/security_admin_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/security_admin_test.rb rename to hbase-shell/src/test/ruby/hbase/security_admin_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/taskmonitor_test.rb b/hbase-shell/src/test/ruby/hbase/taskmonitor_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/taskmonitor_test.rb rename to hbase-shell/src/test/ruby/hbase/taskmonitor_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test.rb b/hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test.rb rename to hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb b/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb deleted file mode 100644 index 0d2f1901438f..000000000000 --- a/hbase-shell/src/test/ruby/no_cluster_tests_runner.rb +++ /dev/null @@ -1,94 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'rubygems' -require 'rake' -require 'set' - - -# This runner will only launch shell tests that don't require a HBase cluster running. - -unless defined?($TEST_CLUSTER) - include Java - - # Set logging level to avoid verboseness - log_level = 'OFF' - org.apache.hadoop.hbase.logging.Log4jUtils.setRootLevel(log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hdfs', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils - .setAllLevels('org.apache.hadoop.ipc.HBaseServer', log_level) - - java_import org.apache.hadoop.hbase.HBaseTestingUtility - - $TEST_CLUSTER = HBaseTestingUtility.new - $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) - $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) - $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6) -end - -require 'test_helper' - -puts "Running tests without a cluster..." - -if java.lang.System.get_property('shell.test.include') - includes = Set.new(java.lang.System.get_property('shell.test.include').split(',')) -end - -if java.lang.System.get_property('shell.test.exclude') - excludes = Set.new(java.lang.System.get_property('shell.test.exclude').split(',')) -end - -files = Dir[ File.dirname(__FILE__) + "/**/*_no_cluster.rb" ] -files.each do |file| - filename = File.basename(file) - if includes != nil && !includes.include?(filename) - puts "Skip #{filename} because of not included" - next - end - if excludes != nil && excludes.include?(filename) - puts "Skip #{filename} because of excluded" - next - end - begin - load(file) - rescue => e - puts "ERROR: #{e}" - raise - end -end - -# If this system property is set, we'll use it to filter the test cases. -runner_args = [] -if java.lang.System.get_property('shell.test') - shell_test_pattern = java.lang.System.get_property('shell.test') - puts "Only running tests that match #{shell_test_pattern}" - runner_args << "--testcase=#{shell_test_pattern}" -end -# first couple of args are to match the defaults, so we can pass options to limit the tests run -if !(Test::Unit::AutoRunner.run(false, nil, runner_args)) - raise "Shell unit tests failed. Check output file for details." -end - -puts "Done with tests! Shutting down the cluster..." -if @own_cluster - $TEST_CLUSTER.shutdownMiniCluster - java.lang.System.exit(0) -end diff --git a/hbase-shell/src/test/ruby/shell/commands_test.rb b/hbase-shell/src/test/ruby/shell/commands_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/commands_test.rb rename to hbase-shell/src/test/ruby/shell/commands_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/converter_test.rb b/hbase-shell/src/test/ruby/shell/converter_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/converter_test.rb rename to hbase-shell/src/test/ruby/shell/converter_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/formatter_test.rb b/hbase-shell/src/test/ruby/shell/formatter_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/formatter_test.rb rename to hbase-shell/src/test/ruby/shell/formatter_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/shell_test.rb b/hbase-shell/src/test/ruby/shell/general_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/shell_test.rb rename to hbase-shell/src/test/ruby/shell/general_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb b/hbase-shell/src/test/ruby/shell/list_locks_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/list_locks_test.rb rename to hbase-shell/src/test/ruby/shell/list_locks_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/list_procedures_test.rb b/hbase-shell/src/test/ruby/shell/list_procedures_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/list_procedures_test.rb rename to hbase-shell/src/test/ruby/shell/list_procedures_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/noninteractive_test.rb b/hbase-shell/src/test/ruby/shell/noninteractive_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/noninteractive_test.rb rename to hbase-shell/src/test/ruby/shell/noninteractive_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb b/hbase-shell/src/test/ruby/shell/sftchange_test_cluster.rb similarity index 100% rename from hbase-shell/src/test/ruby/shell/sftchange_shell_test.rb rename to hbase-shell/src/test/ruby/shell/sftchange_test_cluster.rb diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb index e05d11117e58..4e31b81535a7 100644 --- a/hbase-shell/src/test/ruby/tests_runner.rb +++ b/hbase-shell/src/test/ruby/tests_runner.rb @@ -23,34 +23,13 @@ puts "Ruby description: #{RUBY_DESCRIPTION}" -unless defined?($TEST_CLUSTER) - include Java - - # Set logging level to avoid verboseness - log_level = 'OFF' - org.apache.hadoop.hbase.logging.Log4jUtils.setRootLevel(log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.zookeeper', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hdfs', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils.setAllLevels('org.apache.hadoop.hbase', log_level) - org.apache.hadoop.hbase.logging.Log4jUtils - .setAllLevels('org.apache.hadoop.ipc.HBaseServer', log_level) +require 'test_helper' - java_import org.apache.hadoop.hbase.HBaseTestingUtility +test_suite_name = java.lang.System.get_property('shell.test.suite_name') - $TEST_CLUSTER = HBaseTestingUtility.new - $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) - $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) - $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true") - $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.period', 5000) - $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.delay', 5000) - $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6) - $TEST_CLUSTER.startMiniCluster - @own_cluster = true -end +test_suite_pattern = java.lang.System.get_property('shell.test.suite_pattern') -require 'test_helper' - -puts "Running tests..." +puts "Running tests for #{test_suite_name} with pattern: #{test_suite_pattern} ..." if java.lang.System.get_property('shell.test.include') includes = Set.new(java.lang.System.get_property('shell.test.include').split(',')) @@ -60,7 +39,7 @@ excludes = Set.new(java.lang.System.get_property('shell.test.exclude').split(',')) end -files = Dir[ File.dirname(__FILE__) + "/**/*_test.rb" ] +files = Dir[ File.dirname(__FILE__) + "/" + test_suite_pattern ] files.each do |file| filename = File.basename(file) if includes != nil && !includes.include?(filename) @@ -96,9 +75,3 @@ # Unit tests should not raise uncaught SystemExit exceptions. This could cause tests to be ignored. raise 'Caught SystemExit during unit test execution! Check output file for details.' end - -puts "Done with tests! Shutting down the cluster..." -if @own_cluster - $TEST_CLUSTER.shutdownMiniCluster - java.lang.System.exit(0) -end From 8e332a27b2f8a03a5d47f86a71c68495f9d47b38 Mon Sep 17 00:00:00 2001 From: Hari Krishna Dara Date: Sat, 6 Sep 2025 03:13:56 +0530 Subject: [PATCH 050/336] HBASE-29558: Addendum to fix checkstyle warnings (#7276) (#7277) Signed-off-by: Viraj Jasani --- .../hadoop/hbase/client/AbstractTestShell.java | 18 +++--------------- .../hadoop/hbase/client/TestAdminShell.java | 1 + .../hadoop/hbase/client/TestRSGroupShell.java | 1 + .../hbase/client/TestShellNoCluster.java | 2 ++ 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java index ecd1ea6c5974..e98eecc100ba 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java @@ -18,9 +18,7 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.fs.ErasureCodingUtils; import org.jruby.embed.ScriptingContainer; import org.junit.After; import org.junit.Before; @@ -30,16 +28,17 @@ public abstract class AbstractTestShell implements RubyShellTest { protected final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); protected final ScriptingContainer jruby = new ScriptingContainer(); - protected boolean erasureCodingSupported = false; - + @Override public HBaseTestingUtility getTEST_UTIL() { return TEST_UTIL; } + @Override public ScriptingContainer getJRuby() { return jruby; } + @Override public String getSuitePattern() { return "**/*_test.rb"; } @@ -56,17 +55,6 @@ public void setUp() throws Exception { RubyShellTest.doTestSetup(this); } - protected void setupDFS() throws IOException { - try { - ErasureCodingUtils.enablePolicy(FileSystem.get(TEST_UTIL.getConfiguration()), - "XOR-2-1-1024k"); - erasureCodingSupported = true; - } catch (UnsupportedOperationException e) { - LOG.info( - "Current hadoop version does not support erasure coding, only validation tests will run."); - } - } - @After public void tearDown() throws Exception { TEST_UTIL.shutdownMiniCluster(); diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java index 2622c80ac642..8e644b348c65 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java @@ -44,6 +44,7 @@ public String getIncludeList() { protected boolean erasureCodingSupported = false; + @Override @Before public void setUp() throws Exception { RubyShellTest.setUpConfig(this); diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java index 3f77ca5386da..38b9b2dd002d 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java @@ -35,6 +35,7 @@ public class TestRSGroupShell extends AbstractTestShell { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestRSGroupShell.class); + @Override @Before public void setUp() throws Exception { RubyShellTest.setUpConfig(this); diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java index 5c312ec1dac0..0071b24103b5 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java @@ -31,6 +31,7 @@ public class TestShellNoCluster extends AbstractTestShell { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestShellNoCluster.class); + @Override @Before public void setUp() throws Exception { RubyShellTest.setUpConfig(this); @@ -42,6 +43,7 @@ public void setUp() throws Exception { RubyShellTest.doTestSetup(this); } + @Override @After public void tearDown() throws Exception { // no cluster From 8d51bd39615e792a53fef40b8a395b076be12d06 Mon Sep 17 00:00:00 2001 From: sanjeet006py <36011005+sanjeet006py@users.noreply.github.com> Date: Sat, 6 Sep 2025 03:22:46 +0530 Subject: [PATCH 051/336] HBASE-29494: Capture Scan RPC processing time and queuing time in Scan Metrics (#7278) (#7242) Signed-off-by: Viraj Jasani Signed-off-by: Hari Krishna Dara --- .../client/metrics/ServerSideScanMetrics.java | 10 +++ .../hbase/regionserver/RSRpcServices.java | 52 ++++++++------ .../hbase/regionserver/ScannerContext.java | 16 ++++- .../hbase/client/TestTableScanMetrics.java | 67 ++++++++++++++++++- 4 files changed, 121 insertions(+), 24 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java index 6b3a4f5675ad..eeaea716a443 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java @@ -56,6 +56,8 @@ public void moveToNextRegion() { currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_BLOCK_CACHE_METRIC_NAME); currentRegionScanMetricsData.createCounter(BYTES_READ_FROM_MEMSTORE_METRIC_NAME); currentRegionScanMetricsData.createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); + currentRegionScanMetricsData.createCounter(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + currentRegionScanMetricsData.createCounter(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); } /** @@ -77,6 +79,8 @@ protected AtomicLong createCounter(String counterName) { "BYTES_READ_FROM_BLOCK_CACHE"; public static final String BYTES_READ_FROM_MEMSTORE_METRIC_NAME = "BYTES_READ_FROM_MEMSTORE"; public static final String BLOCK_READ_OPS_COUNT_METRIC_NAME = "BLOCK_READ_OPS_COUNT"; + public static final String RPC_SCAN_PROCESSING_TIME_METRIC_NAME = "RPC_SCAN_PROCESSING_TIME"; + public static final String RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME = "RPC_SCAN_QUEUE_WAIT_TIME"; /** * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0 @@ -121,6 +125,12 @@ protected AtomicLong createCounter(String counterName) { public final AtomicLong blockReadOpsCount = createCounter(BLOCK_READ_OPS_COUNT_METRIC_NAME); + public final AtomicLong rpcScanProcessingTime = + createCounter(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + + public final AtomicLong rpcScanQueueWaitTime = + createCounter(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); + /** * Sets counter with counterName to passed in value, does nothing if counter does not exist. If * region level scan metrics are enabled then sets the value of counter for the current region diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 00ba1233424f..7bad1d99bada 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -3368,7 +3368,8 @@ private void checkLimitOfRows(int numOfCompleteRows, int limitOfRows, boolean mo // return whether we have more results in region. private void scan(HBaseRpcController controller, ScanRequest request, RegionScannerHolder rsh, long maxQuotaResultSize, int maxResults, int limitOfRows, List results, - ScanResponse.Builder builder, RpcCall rpcCall) throws IOException { + ScanResponse.Builder builder, RpcCall rpcCall, ServerSideScanMetrics scanMetrics) + throws IOException { HRegion region = rsh.r; RegionScanner scanner = rsh.s; long maxResultSize; @@ -3421,8 +3422,6 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan final LimitScope timeScope = allowHeartbeatMessages ? LimitScope.BETWEEN_CELLS : LimitScope.BETWEEN_ROWS; - boolean trackMetrics = request.hasTrackScanMetrics() && request.getTrackScanMetrics(); - // Configure with limits for this RPC. Set keep progress true since size progress // towards size limit should be kept between calls to nextRaw ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); @@ -3444,7 +3443,8 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan contextBuilder.setSizeLimit(sizeScope, maxCellSize, maxCellSize, maxBlockSize); contextBuilder.setBatchLimit(scanner.getBatch()); contextBuilder.setTimeLimit(timeScope, timeLimit); - contextBuilder.setTrackMetrics(trackMetrics); + contextBuilder.setTrackMetrics(scanMetrics != null); + contextBuilder.setScanMetrics(scanMetrics); ScannerContext scannerContext = contextBuilder.build(); boolean limitReached = false; long blockBytesScannedBefore = 0; @@ -3565,27 +3565,15 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan builder.setMoreResultsInRegion(moreRows); // Check to see if the client requested that we track metrics server side. If the // client requested metrics, retrieve the metrics from the scanner context. - if (trackMetrics) { + if (scanMetrics != null) { // rather than increment yet another counter in StoreScanner, just set the value here // from block size progress before writing into the response - scannerContext.getMetrics().setCounter( - ServerSideScanMetrics.BLOCK_BYTES_SCANNED_KEY_METRIC_NAME, + scanMetrics.setCounter(ServerSideScanMetrics.BLOCK_BYTES_SCANNED_KEY_METRIC_NAME, scannerContext.getBlockSizeProgress()); if (rpcCall != null) { - scannerContext.getMetrics().setCounter(ServerSideScanMetrics.FS_READ_TIME_METRIC_NAME, + scanMetrics.setCounter(ServerSideScanMetrics.FS_READ_TIME_METRIC_NAME, rpcCall.getFsReadTime()); } - Map metrics = scannerContext.getMetrics().getMetricsMap(); - ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); - NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder(); - - for (Entry entry : metrics.entrySet()) { - pairBuilder.setName(entry.getKey()); - pairBuilder.setValue(entry.getValue()); - metricBuilder.addMetrics(pairBuilder.build()); - } - - builder.setScanMetrics(metricBuilder.build()); } } } finally { @@ -3721,6 +3709,8 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque boolean scannerClosed = false; try { List results = new ArrayList<>(Math.min(rows, 512)); + boolean trackMetrics = request.hasTrackScanMetrics() && request.getTrackScanMetrics(); + ServerSideScanMetrics scanMetrics = trackMetrics ? new ServerSideScanMetrics() : null; if (rows > 0) { boolean done = false; // Call coprocessor. Get region info from scanner. @@ -3740,7 +3730,7 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque } if (!done) { scan((HBaseRpcController) controller, request, rsh, maxQuotaResultSize, rows, limitOfRows, - results, builder, rpcCall); + results, builder, rpcCall, scanMetrics); } else { builder.setMoreResultsInRegion(!results.isEmpty()); } @@ -3792,6 +3782,28 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque throw new TimeoutIOException("Client deadline exceeded, cannot return results"); } + if (scanMetrics != null) { + if (rpcCall != null) { + long rpcScanTime = EnvironmentEdgeManager.currentTime() - rpcCall.getStartTime(); + long rpcQueueWaitTime = rpcCall.getStartTime() - rpcCall.getReceiveTime(); + scanMetrics.addToCounter(ServerSideScanMetrics.RPC_SCAN_PROCESSING_TIME_METRIC_NAME, + rpcScanTime); + scanMetrics.addToCounter(ServerSideScanMetrics.RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME, + rpcQueueWaitTime); + } + Map metrics = scanMetrics.getMetricsMap(); + ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); + NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder(); + + for (Entry entry : metrics.entrySet()) { + pairBuilder.setName(entry.getKey()); + pairBuilder.setValue(entry.getValue()); + metricBuilder.addMetrics(pairBuilder.build()); + } + + builder.setScanMetrics(metricBuilder.build()); + } + return builder.build(); } catch (IOException e) { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java index 09945ca23030..06ac1f44b857 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java @@ -124,6 +124,11 @@ public class ScannerContext { final ServerSideScanMetrics metrics; ScannerContext(boolean keepProgress, LimitFields limitsToCopy, boolean trackMetrics) { + this(keepProgress, limitsToCopy, trackMetrics, null); + } + + ScannerContext(boolean keepProgress, LimitFields limitsToCopy, boolean trackMetrics, + ServerSideScanMetrics scanMetrics) { this.limits = new LimitFields(); if (limitsToCopy != null) { this.limits.copy(limitsToCopy); @@ -134,7 +139,8 @@ public class ScannerContext { this.keepProgress = keepProgress; this.scannerState = DEFAULT_STATE; - this.metrics = trackMetrics ? new ServerSideScanMetrics() : null; + this.metrics = + trackMetrics ? (scanMetrics != null ? scanMetrics : new ServerSideScanMetrics()) : null; } public boolean isTrackingMetrics() { @@ -449,6 +455,7 @@ public static final class Builder { boolean keepProgress = DEFAULT_KEEP_PROGRESS; boolean trackMetrics = false; LimitFields limits = new LimitFields(); + ServerSideScanMetrics scanMetrics = null; private Builder() { } @@ -487,8 +494,13 @@ public Builder setBatchLimit(int batchLimit) { return this; } + public Builder setScanMetrics(ServerSideScanMetrics scanMetrics) { + this.scanMetrics = scanMetrics; + return this; + } + public ScannerContext build() { - return new ScannerContext(keepProgress, limits, trackMetrics); + return new ScannerContext(keepProgress, limits, trackMetrics, scanMetrics); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableScanMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableScanMetrics.java index 6f84ddcc314a..ac3b98a6bf78 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableScanMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableScanMetrics.java @@ -23,6 +23,8 @@ import static org.apache.hadoop.hbase.client.metrics.ScanMetrics.REGIONS_SCANNED_METRIC_NAME; import static org.apache.hadoop.hbase.client.metrics.ScanMetrics.RPC_RETRIES_METRIC_NAME; import static org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME; +import static org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics.RPC_SCAN_PROCESSING_TIME_METRIC_NAME; +import static org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics.RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -36,16 +38,20 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.client.metrics.ScanMetricsRegionInfo; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FutureUtils; import org.junit.AfterClass; @@ -57,7 +63,7 @@ import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; -@Category({ ClientTests.class, MediumTests.class }) +@Category({ ClientTests.class, LargeTests.class }) public class TestTableScanMetrics extends FromClientSideBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = @@ -334,6 +340,8 @@ public void run() { Map metricsMap = entry.getValue(); // Remove millis between nexts metric as it is not deterministic metricsMap.remove(MILLIS_BETWEEN_NEXTS_METRIC_NAME); + metricsMap.remove(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + metricsMap.remove(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); Assert.assertNotNull(scanMetricsRegionInfo.getEncodedRegionName()); Assert.assertNotNull(scanMetricsRegionInfo.getServerName()); Assert.assertEquals(1, (long) metricsMap.get(REGIONS_SCANNED_METRIC_NAME)); @@ -351,6 +359,59 @@ public void run() { } } + @Test + public void testRPCCallProcessingAndQueueWaitTimeMetrics() throws Exception { + final int numThreads = 20; + Configuration conf = TEST_UTIL.getConfiguration(); + // Handler count is 3 by default. + int handlerCount = conf.getInt(HConstants.REGION_SERVER_HANDLER_COUNT, + HConstants.DEFAULT_REGION_SERVER_HANDLER_COUNT); + // Keep the number of threads to be high enough for RPC calls to queue up. For now going with 6 + // times the handler count. + Assert.assertTrue(numThreads > 6 * handlerCount); + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads); + TableName tableName = TableName.valueOf( + TestTableScanMetrics.class.getSimpleName() + "_testRPCCallProcessingAndQueueWaitTimeMetrics"); + AtomicLong totalScanRpcTime = new AtomicLong(0); + AtomicLong totalQueueWaitTime = new AtomicLong(0); + CountDownLatch latch = new CountDownLatch(numThreads); + try (Table table = TEST_UTIL.createMultiRegionTable(tableName, CF)) { + TEST_UTIL.loadTable(table, CF); + for (int i = 0; i < numThreads; i++) { + executor.execute(new Runnable() { + @Override + public void run() { + try { + Scan scan = generateScan(EMPTY_BYTE_ARRAY, EMPTY_BYTE_ARRAY); + scan.setEnableScanMetricsByRegion(true); + scan.setCaching(2); + try (ResultScanner rs = table.getScanner(scan)) { + Result r; + while ((r = rs.next()) != null) { + Assert.assertFalse(r.isEmpty()); + } + ScanMetrics scanMetrics = rs.getScanMetrics(); + Map metricsMap = scanMetrics.getMetricsMap(); + totalScanRpcTime.addAndGet(metricsMap.get(RPC_SCAN_PROCESSING_TIME_METRIC_NAME)); + totalQueueWaitTime.addAndGet(metricsMap.get(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME)); + } + latch.countDown(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + latch.await(); + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + Assert.assertTrue(totalScanRpcTime.get() > 0); + Assert.assertTrue(totalQueueWaitTime.get() > 0); + } finally { + TEST_UTIL.deleteTable(tableName); + } + } + @Test public void testScanMetricsByRegionWithRegionMove() throws Exception { TableName tableName = TableName.valueOf( @@ -591,6 +652,8 @@ private void mergeScanMetricsByRegion(Map metricsMap = entry.getValue(); // Remove millis between nexts metric as it is not deterministic metricsMap.remove(MILLIS_BETWEEN_NEXTS_METRIC_NAME); + metricsMap.remove(RPC_SCAN_PROCESSING_TIME_METRIC_NAME); + metricsMap.remove(RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME); if (dstMap.containsKey(scanMetricsRegionInfo)) { Map dstMetricsMap = dstMap.get(scanMetricsRegionInfo); for (Map.Entry metricEntry : metricsMap.entrySet()) { From 264177d4cef7a8377eeb438da7de6962bbe1b754 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Sat, 6 Sep 2025 08:22:47 +0200 Subject: [PATCH 052/336] HBASE-23671 Upgrade to JUnit 5 (#7280) also update maven-surefire-plugin to 3.5.3 Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang Reviewed-by: Aman Poonia (cherry picked from commit e4efbda3091b393ab5a1eb7ea5e010c53f6c15ec) --- hbase-annotations/pom.xml | 7 ++++ hbase-archetypes/hbase-client-project/pom.xml | 19 ++++++++- .../hbase-shaded-client-project/pom.xml | 19 ++++++++- hbase-assembly/pom.xml | 20 ++++++++- hbase-asyncfs/pom.xml | 19 ++++++++- hbase-backup/pom.xml | 19 ++++++++- hbase-checkstyle/pom.xml | 7 ++++ hbase-client/pom.xml | 19 ++++++++- hbase-common/pom.xml | 19 ++++++++- hbase-endpoint/pom.xml | 19 ++++++++- hbase-examples/pom.xml | 19 ++++++++- hbase-extensions/hbase-openssl/pom.xml | 7 ++++ hbase-external-blockcache/pom.xml | 19 ++++++++- hbase-hadoop-compat/pom.xml | 19 ++++++++- hbase-hadoop2-compat/pom.xml | 19 ++++++++- hbase-hbtop/pom.xml | 20 +++++++++ hbase-http/pom.xml | 19 ++++++++- hbase-it/pom.xml | 19 ++++++++- hbase-logging/pom.xml | 19 ++++++++- hbase-mapreduce/pom.xml | 19 ++++++++- hbase-metrics-api/pom.xml | 19 ++++++++- hbase-metrics/pom.xml | 19 ++++++++- hbase-procedure/pom.xml | 19 ++++++++- hbase-protocol-shaded/pom.xml | 19 ++++++++- hbase-protocol/pom.xml | 18 +++----- hbase-replication/pom.xml | 19 ++++++++- hbase-rest/pom.xml | 19 ++++++++- hbase-rsgroup/pom.xml | 18 +++++++- hbase-server/pom.xml | 19 ++++++++- .../hbase-shaded-check-invariants/pom.xml | 19 ++++++++- .../hbase-shaded-testing-util-tester/pom.xml | 19 ++++++++- .../pom.xml | 19 ++++++++- hbase-shell/pom.xml | 19 ++++++++- hbase-thrift/pom.xml | 19 ++++++++- hbase-zookeeper/pom.xml | 19 ++++++++- pom.xml | 41 +++++++++++-------- 36 files changed, 581 insertions(+), 89 deletions(-) diff --git a/hbase-annotations/pom.xml b/hbase-annotations/pom.xml index 2dc7c6102722..d977f3c78c6f 100644 --- a/hbase-annotations/pom.xml +++ b/hbase-annotations/pom.xml @@ -40,6 +40,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/hbase-archetypes/hbase-client-project/pom.xml b/hbase-archetypes/hbase-client-project/pom.xml index 6d70cb345275..88115c1d552c 100644 --- a/hbase-archetypes/hbase-client-project/pom.xml +++ b/hbase-archetypes/hbase-client-project/pom.xml @@ -87,8 +87,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-archetypes/hbase-shaded-client-project/pom.xml b/hbase-archetypes/hbase-shaded-client-project/pom.xml index 69832f8f6326..8f6105421b9e 100644 --- a/hbase-archetypes/hbase-shaded-client-project/pom.xml +++ b/hbase-archetypes/hbase-shaded-client-project/pom.xml @@ -87,8 +87,23 @@ runtime - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 459d3d2cb922..f2080226f0cb 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -388,8 +388,24 @@ hbase-rsgroup - junit - junit + org.junit.jupiter + junit-jupiter-api + compile + + + org.junit.jupiter + junit-jupiter-engine + compile + + + org.junit.jupiter + junit-jupiter-params + compile + + + org.junit.vintage + junit-vintage-engine + compile org.mockito diff --git a/hbase-asyncfs/pom.xml b/hbase-asyncfs/pom.xml index aa9787292eb6..a4b205b784a6 100644 --- a/hbase-asyncfs/pom.xml +++ b/hbase-asyncfs/pom.xml @@ -69,8 +69,23 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml index 7bbf59b1e96a..2eb8025bc1b3 100644 --- a/hbase-backup/pom.xml +++ b/hbase-backup/pom.xml @@ -163,8 +163,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-checkstyle/pom.xml b/hbase-checkstyle/pom.xml index bd8d5f7e80bb..67fc77a4dc5f 100644 --- a/hbase-checkstyle/pom.xml +++ b/hbase-checkstyle/pom.xml @@ -43,6 +43,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + org.apache.maven.plugins maven-site-plugin diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 8343037a233d..54a3b2baed07 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -173,8 +173,23 @@ metrics-core - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 6f3cdf66eede..9222eca81b62 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -110,8 +110,23 @@ commons-crypto - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml index f4fdfd4f7fc6..840606b5b4a7 100644 --- a/hbase-endpoint/pom.xml +++ b/hbase-endpoint/pom.xml @@ -164,8 +164,23 @@ slf4j-api - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index aa84fdf3ceb7..e5b2566c293e 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -134,8 +134,23 @@ hbase-rest - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-extensions/hbase-openssl/pom.xml b/hbase-extensions/hbase-openssl/pom.xml index bbe161921cc6..b1183a1bec76 100644 --- a/hbase-extensions/hbase-openssl/pom.xml +++ b/hbase-extensions/hbase-openssl/pom.xml @@ -49,6 +49,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index b1afadc9b409..d1f189e187a5 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -102,8 +102,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml index 059ad7b1c1fb..993cfc39f0fc 100644 --- a/hbase-hadoop-compat/pom.xml +++ b/hbase-hadoop-compat/pom.xml @@ -65,8 +65,23 @@ hbase-metrics-api - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index 0c3d64509be0..941dca3c6086 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -42,8 +42,23 @@ limitations under the License. hbase-hadoop-compat - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-hbtop/pom.xml b/hbase-hbtop/pom.xml index 370b17074f7f..b9d9c376faa2 100644 --- a/hbase-hbtop/pom.xml +++ b/hbase-hbtop/pom.xml @@ -48,6 +48,26 @@ org.slf4j slf4j-api + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + test + org.mockito mockito-core diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 47373b25e779..7dfdd25dc64f 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -116,8 +116,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index e75219d1ed1d..743180e45b43 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -186,8 +186,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-logging/pom.xml b/hbase-logging/pom.xml index dd79fb394c97..a1521eb750ba 100644 --- a/hbase-logging/pom.xml +++ b/hbase-logging/pom.xml @@ -43,8 +43,23 @@ slf4j-api - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml index 40b6cae60da4..644c819c33a9 100644 --- a/hbase-mapreduce/pom.xml +++ b/hbase-mapreduce/pom.xml @@ -201,8 +201,23 @@ zookeeper - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-metrics-api/pom.xml b/hbase-metrics-api/pom.xml index 9c8d9dc79b4a..5c6e65f66fc7 100644 --- a/hbase-metrics-api/pom.xml +++ b/hbase-metrics-api/pom.xml @@ -74,8 +74,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-metrics/pom.xml b/hbase-metrics/pom.xml index 5fe3d26147ff..cefc26829359 100644 --- a/hbase-metrics/pom.xml +++ b/hbase-metrics/pom.xml @@ -83,8 +83,23 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index caa037c21510..e7985d476ef7 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -77,8 +77,23 @@ true - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index b742b35b2fb7..e96405ae5c8e 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -42,8 +42,23 @@ hbase-shaded-protobuf - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml index bfa77441d531..9d9bcb6139c0 100644 --- a/hbase-protocol/pom.xml +++ b/hbase-protocol/pom.xml @@ -64,20 +64,12 @@ + + org.apache.maven.plugins maven-surefire-plugin - - - - secondPartTestsExecution - - test - - test - - true - - - + + true + org.xolstice.maven.plugins diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml index 531a4bbfb9e2..c80b89431e69 100644 --- a/hbase-replication/pom.xml +++ b/hbase-replication/pom.xml @@ -100,8 +100,23 @@ zookeeper - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 7bd7b188cc32..032d0eccd2b1 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -200,8 +200,23 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml index 10bfb0c520c4..1ff92ba67450 100644 --- a/hbase-rsgroup/pom.xml +++ b/hbase-rsgroup/pom.xml @@ -161,10 +161,24 @@ test - junit - junit + org.junit.jupiter + junit-jupiter-api test + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine + diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 5b7a6aa09ed6..ea332ba5d609 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -293,8 +293,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml index e8f1617da3bb..a203e5b8dd64 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml @@ -70,8 +70,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + provided + + + org.junit.jupiter + junit-jupiter-engine + provided + + + org.junit.jupiter + junit-jupiter-params + provided + + + org.junit.vintage + junit-vintage-engine provided diff --git a/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml index 9c870afde1f6..d387ed151de6 100644 --- a/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml +++ b/hbase-shaded/hbase-shaded-testing-util-tester/pom.xml @@ -34,8 +34,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml index 0fa54bbb958f..8f6ceb1a0bb3 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml @@ -60,8 +60,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + provided + + + org.junit.jupiter + junit-jupiter-engine + provided + + + org.junit.jupiter + junit-jupiter-params + provided + + + org.junit.vintage + junit-vintage-engine provided diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 30e51776df1b..25ec73039df6 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -80,8 +80,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index ac9eda4d4436..558b6f1e1318 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -98,8 +98,23 @@ libthrift - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/hbase-zookeeper/pom.xml b/hbase-zookeeper/pom.xml index f2b997b09f10..d528e50c2610 100644 --- a/hbase-zookeeper/pom.xml +++ b/hbase-zookeeper/pom.xml @@ -109,8 +109,23 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.vintage + junit-vintage-engine test diff --git a/pom.xml b/pom.xml index c8f3e0f3272d..97de29083911 100644 --- a/pom.xml +++ b/pom.xml @@ -596,7 +596,8 @@ 2.1.1 9.0.104 9.3.15.0 - 4.13.2 + 5.13.4 + 5.13.4 1.3 1.49.0 1.29.0-alpha @@ -658,7 +659,7 @@ 1.3.9-1 4.7.3 4.7.2.1 - 3.1.0 + 3.5.3 2.12 1.0.1 2.44.4 @@ -1372,9 +1373,28 @@ - junit - junit - ${junit.version} + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + org.junit.jupiter + junit-jupiter-engine + ${junit.jupiter.version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit.jupiter.version} + test + + + org.junit.vintage + junit-vintage-engine + ${junit.vintage.version} + test org.hamcrest @@ -1600,17 +1620,6 @@ - - - - junit - junit - test - - false false @@ -1707,16 +1706,6 @@ - - - - org.apache.maven.surefire - ${surefire.provider} - ${surefire.version} - - secondPartTestsExecution @@ -4960,7 +4949,6 @@ - surefire-junit4 false true From 8cf548fceec7e027671a7234578b5b8a66a7c707 Mon Sep 17 00:00:00 2001 From: wangxiangdong123 <810410559@qq.com> Date: Tue, 9 Sep 2025 20:54:17 +0800 Subject: [PATCH 056/336] HBASE-29571 Fix Javadoc typo: 'repoen' should be 'reopen' (#7273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dávid Paksy (cherry picked from commit 5629108a44ce6e52bc85fa1522e3bb88cc875202) --- .../hbase/master/assignment/MergeTableRegionsProcedure.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 6e74cb8bc9bd..8203a3458372 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -672,8 +672,8 @@ private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOExceptio **/ private void rollbackCloseRegionsForMerge(MasterProcedureEnv env) throws IOException { // At this point we should check if region was actually closed. If it was not closed then we - // don't need to repoen the region and we can just change the regionNode state to OPEN. - // if it is alredy closed then we need to do a reopen of region + // don't need to reopen the region and we can just change the regionNode state to OPEN. + // if it is already closed then we need to do a reopen of region List toAssign = new ArrayList<>(); for (RegionInfo rinfo : regionsToMerge) { RegionStateNode regionStateNode = From 313b9cc799a0c37b735664f79761bc930728c803 Mon Sep 17 00:00:00 2001 From: Diya Maria Abraham <93218556+DiyaMariaAbraham@users.noreply.github.com> Date: Thu, 11 Sep 2025 20:21:53 +0530 Subject: [PATCH 057/336] HBASE-29566: TestPrefetch.testPrefetchWithDelay seems flakey (#7287) Signed-off-by: Wellington Chevreuil --- .../hadoop/hbase/io/hfile/TestPrefetch.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index 8e278e40336e..c9966745d717 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MatcherPredicate; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -326,16 +327,14 @@ public void testPrefetchWithDelay() throws Exception { // Wait for 20 seconds, no thread should start prefetch Thread.sleep(20000); assertFalse("Prefetch threads should not be running at this point", reader.prefetchStarted()); - while (!reader.prefetchStarted()) { - assertTrue("Prefetch delay has not been expired yet", - getElapsedTime(startTime) < PrefetchExecutor.getPrefetchDelay()); - } - if (reader.prefetchStarted()) { - // Added some delay as we have started the timer a bit late. - Thread.sleep(500); - assertTrue("Prefetch should start post configured delay", - getElapsedTime(startTime) > PrefetchExecutor.getPrefetchDelay()); - } + long timeout = 10000; + Waiter.waitFor(conf, 10000, () -> (reader.prefetchStarted() || reader.prefetchComplete())); + + assertTrue(reader.prefetchStarted() || reader.prefetchComplete()); + + assertTrue("Prefetch should start post configured delay", + getElapsedTime(startTime) > PrefetchExecutor.getPrefetchDelay()); + conf.setInt(PREFETCH_DELAY, 1000); conf.setFloat(PREFETCH_DELAY_VARIATION, PREFETCH_DELAY_VARIATION_DEFAULT_VALUE); prefetchExecutorNotifier.onConfigurationChange(conf); From 45edf3d96f6cadea929f5fb0e4521817ed6de6ac Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 12 Sep 2025 10:52:49 -0400 Subject: [PATCH 058/336] HBASE-29573: Fully load QuotaCache instead of reading individual rows on demand (#7282) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/QuotaTableUtil.java | 31 -- .../hadoop/hbase/quotas/QuotaCache.java | 302 +++++++----------- .../hadoop/hbase/quotas/QuotaState.java | 38 +-- .../apache/hadoop/hbase/quotas/QuotaUtil.java | 163 +++++----- .../hadoop/hbase/quotas/UserQuotaState.java | 22 +- .../hbase/quotas/TestAtomicReadQuota.java | 1 - .../quotas/TestBlockBytesScannedQuota.java | 1 - .../quotas/TestClusterScopeQuotaThrottle.java | 1 - .../hbase/quotas/TestDefaultAtomicQuota.java | 1 - .../quotas/TestDefaultHandlerUsageQuota.java | 1 - .../hadoop/hbase/quotas/TestDefaultQuota.java | 7 +- .../hadoop/hbase/quotas/TestQuotaCache.java | 40 +-- .../hadoop/hbase/quotas/TestQuotaCache2.java | 130 ++++++++ .../hadoop/hbase/quotas/TestQuotaState.java | 58 +--- .../hbase/quotas/TestQuotaThrottle.java | 1 - .../hbase/quotas/TestQuotaUserOverride.java | 1 - .../quotas/TestThreadHandlerUsageQuota.java | 8 +- 17 files changed, 362 insertions(+), 444 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 1afb15c0ac61..4bdf5e5af049 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -206,37 +206,6 @@ private static Quotas getQuotas(final Connection connection, final byte[] rowKey return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier)); } - public static Get makeGetForTableQuotas(final TableName table) { - Get get = new Get(getTableRowKey(table)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForNamespaceQuotas(final String namespace) { - Get get = new Get(getNamespaceRowKey(namespace)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForRegionServerQuotas(final String regionServer) { - Get get = new Get(getRegionServerRowKey(regionServer)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForUserQuotas(final String user, final Iterable tables, - final Iterable namespaces) { - Get get = new Get(getUserRowKey(user)); - get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); - for (final TableName table : tables) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table)); - } - for (final String ns : namespaces) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns)); - } - return get; - } - public static Scan makeScan(final QuotaFilter filter) { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 2ec9d049f7da..16681eb45f8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -19,30 +19,23 @@ import java.io.IOException; import java.time.Duration; -import java.util.ArrayList; import java.util.EnumSet; -import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -73,18 +66,15 @@ public class QuotaCache implements Stoppable { public static final String QUOTA_USER_REQUEST_ATTRIBUTE_OVERRIDE_KEY = "hbase.quota.user.override.key"; private static final int REFRESH_DEFAULT_PERIOD = 43_200_000; // 12 hours - private static final int EVICT_PERIOD_FACTOR = 5; - // for testing purpose only, enforce the cache to be always refreshed - static boolean TEST_FORCE_REFRESH = false; - // for testing purpose only, block cache refreshes to reliably verify state - static boolean TEST_BLOCK_REFRESH = false; + private final Object initializerLock = new Object(); + private volatile boolean initialized = false; + + private volatile Map namespaceQuotaCache = new HashMap<>(); + private volatile Map tableQuotaCache = new HashMap<>(); + private volatile Map userQuotaCache = new HashMap<>(); + private volatile Map regionServerQuotaCache = new HashMap<>(); - private final ConcurrentMap namespaceQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap tableQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap userQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap regionServerQuotaCache = - new ConcurrentHashMap<>(); private volatile boolean exceedThrottleQuotaEnabled = false; // factors used to divide cluster scope quota into machine scope quota private volatile double machineQuotaFactor = 1; @@ -96,62 +86,6 @@ public class QuotaCache implements Stoppable { private QuotaRefresherChore refreshChore; private boolean stopped = true; - private final Fetcher userQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String user) { - final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); - final Set tables = QuotaCache.this.tableQuotaCache.keySet(); - return QuotaUtil.makeGetForUserQuotas(user, tables, namespaces); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, tableMachineQuotaFactors, - machineQuotaFactor); - } - }; - - private final Fetcher regionServerQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String regionServer) { - return QuotaUtil.makeGetForRegionServerQuotas(regionServer); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); - } - }; - - private final Fetcher tableQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final TableName table) { - return QuotaUtil.makeGetForTableQuotas(table); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors); - } - }; - - private final Fetcher namespaceQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String namespace) { - return QuotaUtil.makeGetForNamespaceQuotas(namespace); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, machineQuotaFactor); - } - }; - public QuotaCache(final RegionServerServices rsServices) { this.rsServices = rsServices; this.userOverrideRequestAttributeKey = @@ -163,10 +97,8 @@ public void start() throws IOException { Configuration conf = rsServices.getConfiguration(); // Refresh the cache every 12 hours, and every time a quota is changed, and every time a - // configuration - // reload is triggered. Periodic reloads are kept to a minimum to avoid flooding the - // RegionServer - // holding the hbase:quota table with requests. + // configuration reload is triggered. Periodic reloads are kept to a minimum to avoid + // flooding the RegionServer holding the hbase:quota table with requests. int period = conf.getInt(REFRESH_CONF_KEY, REFRESH_DEFAULT_PERIOD); refreshChore = new QuotaRefresherChore(conf, period, this); rsServices.getChoreService().scheduleChore(refreshChore); @@ -186,6 +118,34 @@ public boolean isStopped() { return stopped; } + private void ensureInitialized() { + if (!initialized) { + synchronized (initializerLock) { + if (!initialized) { + refreshChore.chore(); + initialized = true; + } + } + } + } + + private Map fetchUserQuotaStateEntries() throws IOException { + return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), tableMachineQuotaFactors, + machineQuotaFactor); + } + + private Map fetchRegionServerQuotaStateEntries() throws IOException { + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection()); + } + + private Map fetchTableQuotaStateEntries() throws IOException { + return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), tableMachineQuotaFactors); + } + + private Map fetchNamespaceQuotaStateEntries() throws IOException { + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), machineQuotaFactor); + } + /** * Returns the limiter associated to the specified user/table. * @param ugi the user to limit @@ -206,12 +166,13 @@ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableNa */ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { String user = getQuotaUserName(ugi); - if (!userQuotaCache.containsKey(user)) { - userQuotaCache.put(user, - QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); - fetch("user", userQuotaCache, userQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to userQuotaCache + Map cache = userQuotaCache; + if (!cache.containsKey(user)) { + cache.put(user, QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration())); } - return userQuotaCache.get(user); + return cache.get(user); } /** @@ -220,11 +181,13 @@ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { * @return the limiter associated to the specified table */ public QuotaLimiter getTableLimiter(final TableName table) { - if (!tableQuotaCache.containsKey(table)) { - tableQuotaCache.put(table, new QuotaState()); - fetch("table", tableQuotaCache, tableQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to tableQuotaCache + Map cache = tableQuotaCache; + if (!cache.containsKey(table)) { + cache.put(table, new QuotaState()); } - return tableQuotaCache.get(table).getGlobalLimiter(); + return cache.get(table).getGlobalLimiter(); } /** @@ -233,11 +196,13 @@ public QuotaLimiter getTableLimiter(final TableName table) { * @return the limiter associated to the specified namespace */ public QuotaLimiter getNamespaceLimiter(final String namespace) { - if (!namespaceQuotaCache.containsKey(namespace)) { - namespaceQuotaCache.put(namespace, new QuotaState()); - fetch("namespace", namespaceQuotaCache, namespaceQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to namespaceQuotaCache + Map cache = namespaceQuotaCache; + if (!cache.containsKey(namespace)) { + cache.put(namespace, new QuotaState()); } - return namespaceQuotaCache.get(namespace).getGlobalLimiter(); + return cache.get(namespace).getGlobalLimiter(); } /** @@ -246,41 +211,19 @@ public QuotaLimiter getNamespaceLimiter(final String namespace) { * @return the limiter associated to the specified region server */ public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) { - if (!regionServerQuotaCache.containsKey(regionServer)) { - regionServerQuotaCache.put(regionServer, new QuotaState()); - fetch("regionServer", regionServerQuotaCache, regionServerQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to regionServerQuotaCache + Map cache = regionServerQuotaCache; + if (!cache.containsKey(regionServer)) { + cache.put(regionServer, new QuotaState()); } - return regionServerQuotaCache.get(regionServer).getGlobalLimiter(); + return cache.get(regionServer).getGlobalLimiter(); } protected boolean isExceedThrottleQuotaEnabled() { return exceedThrottleQuotaEnabled; } - private void fetch(final String type, final Map quotasMap, - final Fetcher fetcher) { - // Find the quota entries to update - List gets = quotasMap.keySet().stream().map(fetcher::makeGet).collect(Collectors.toList()); - - // fetch and update the quota entries - if (!gets.isEmpty()) { - try { - for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { - V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); - if (quotaInfo != null) { - quotaInfo.update(entry.getValue()); - } - - if (LOG.isTraceEnabled()) { - LOG.trace("Loading {} key={} quotas={}", type, entry.getKey(), quotaInfo); - } - } - } catch (IOException e) { - LOG.warn("Unable to read {} from quota table", type, e); - } - } - } - /** * Applies a request attribute user override if available, otherwise returns the UGI's short * username @@ -311,18 +254,22 @@ void forceSynchronousCacheRefresh() { refreshChore.chore(); } + /** visible for testing */ Map getNamespaceQuotaCache() { return namespaceQuotaCache; } + /** visible for testing */ Map getRegionServerQuotaCache() { return regionServerQuotaCache; } + /** visible for testing */ Map getTableQuotaCache() { return tableQuotaCache; } + /** visible for testing */ Map getUserQuotaCache() { return userQuotaCache; } @@ -359,38 +306,44 @@ public synchronized boolean triggerNow() { } @Override - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "GC_UNRELATED_TYPES", - justification = "I do not understand why the complaints, it looks good to me -- FIX") protected void chore() { - while (TEST_BLOCK_REFRESH) { - LOG.info("TEST_BLOCK_REFRESH=true, so blocking QuotaCache refresh until it is false"); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + updateQuotaFactors(); + + try { + Map newUserQuotaCache = new HashMap<>(fetchUserQuotaStateEntries()); + updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); + userQuotaCache = newUserQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching user quotas", e); } - // Prefetch online tables/namespaces - for (TableName table : ((HRegionServer) QuotaCache.this.rsServices).getOnlineTables()) { - if (table.isSystemTable()) { - continue; - } - QuotaCache.this.tableQuotaCache.computeIfAbsent(table, key -> new QuotaState()); - final String ns = table.getNamespaceAsString(); + try { + Map newRegionServerQuotaCache = + new HashMap<>(fetchRegionServerQuotaStateEntries()); + updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); + regionServerQuotaCache = newRegionServerQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching region server quotas", e); + } - QuotaCache.this.namespaceQuotaCache.computeIfAbsent(ns, key -> new QuotaState()); + try { + Map newTableQuotaCache = + new HashMap<>(fetchTableQuotaStateEntries()); + updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); + tableQuotaCache = newTableQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing table quotas", e); } - QuotaCache.this.regionServerQuotaCache - .computeIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, key -> new QuotaState()); + try { + Map newNamespaceQuotaCache = + new HashMap<>(fetchNamespaceQuotaStateEntries()); + updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); + namespaceQuotaCache = newNamespaceQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing namespace quotas", e); + } - updateQuotaFactors(); - fetchAndEvict("namespace", QuotaCache.this.namespaceQuotaCache, namespaceQuotaStateFetcher); - fetchAndEvict("table", QuotaCache.this.tableQuotaCache, tableQuotaStateFetcher); - fetchAndEvict("user", QuotaCache.this.userQuotaCache, userQuotaStateFetcher); - fetchAndEvict("regionServer", QuotaCache.this.regionServerQuotaCache, - regionServerQuotaStateFetcher); fetchExceedThrottleQuota(); } @@ -403,48 +356,6 @@ private void fetchExceedThrottleQuota() { } } - private void fetchAndEvict(final String type, - final ConcurrentMap quotasMap, final Fetcher fetcher) { - long now = EnvironmentEdgeManager.currentTime(); - long evictPeriod = getPeriod() * EVICT_PERIOD_FACTOR; - // Find the quota entries to update - List gets = new ArrayList<>(); - List toRemove = new ArrayList<>(); - for (Map.Entry entry : quotasMap.entrySet()) { - long lastQuery = entry.getValue().getLastQuery(); - if (lastQuery > 0 && (now - lastQuery) >= evictPeriod) { - toRemove.add(entry.getKey()); - } else { - gets.add(fetcher.makeGet(entry.getKey())); - } - } - - for (final K key : toRemove) { - if (LOG.isTraceEnabled()) { - LOG.trace("evict " + type + " key=" + key); - } - quotasMap.remove(key); - } - - // fetch and update the quota entries - if (!gets.isEmpty()) { - try { - for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { - V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); - if (quotaInfo != null) { - quotaInfo.update(entry.getValue()); - } - - if (LOG.isTraceEnabled()) { - LOG.trace("refresh " + type + " key=" + entry.getKey() + " quotas=" + quotaInfo); - } - } - } catch (IOException e) { - LOG.warn("Unable to read " + type + " from quota table", e); - } - } - } - /** * Update quota factors which is used to divide cluster scope quota into machine scope quota For * user/namespace/user over namespace quota, use [1 / RSNum] as machine factor. For table/user @@ -520,6 +431,20 @@ private void updateMachineQuotaFactors(int rsSize) { } } + /** visible for testing */ + static void updateNewCacheFromOld(Map oldCache, + Map newCache) { + for (Map.Entry entry : oldCache.entrySet()) { + K key = entry.getKey(); + if (newCache.containsKey(key)) { + V newState = newCache.get(key); + V oldState = entry.getValue(); + oldState.update(newState); + newCache.put(key, oldState); + } + } + } + static class RefreshableExpiringValueCache { private final String name; private final LoadingCache> cache; @@ -560,9 +485,4 @@ static interface ThrowingSupplier { T get() throws Exception; } - interface Fetcher { - Get makeGet(Key key); - - Map fetchEntries(List gets) throws IOException; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java index 7c9445e15587..61aa9d7f068f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -32,33 +31,14 @@ justification = "FindBugs seems confused; says globalLimiter and lastUpdate " + "are mostly synchronized...but to me it looks like they are totally synchronized") public class QuotaState { - protected long lastUpdate = 0; - protected long lastQuery = 0; - protected QuotaLimiter globalLimiter = NoopQuotaLimiter.get(); - public QuotaState() { - this(0); - } - - public QuotaState(final long updateTs) { - lastUpdate = updateTs; - } - - public synchronized long getLastUpdate() { - return lastUpdate; - } - - public synchronized long getLastQuery() { - return lastQuery; - } - @Override public synchronized String toString() { StringBuilder builder = new StringBuilder(); - builder.append("QuotaState(ts=" + getLastUpdate()); + builder.append("QuotaState("); if (isBypass()) { - builder.append(" bypass"); + builder.append("bypass"); } else { if (globalLimiter != NoopQuotaLimiter.get()) { // builder.append(" global-limiter"); @@ -85,6 +65,11 @@ public synchronized void setQuotas(final Quotas quotas) { } } + /** visible for testing */ + void setGlobalLimiter(QuotaLimiter globalLimiter) { + this.globalLimiter = globalLimiter; + } + /** * Perform an update of the quota info based on the other quota info object. (This operation is * executed by the QuotaCache) @@ -97,7 +82,6 @@ public synchronized void update(final QuotaState other) { } else { globalLimiter = QuotaLimiterFactory.update(globalLimiter, other.globalLimiter); } - lastUpdate = other.lastUpdate; } /** @@ -105,15 +89,7 @@ public synchronized void update(final QuotaState other) { * @return the quota limiter */ public synchronized QuotaLimiter getGlobalLimiter() { - lastQuery = EnvironmentEdgeManager.currentTime(); return globalLimiter; } - /** - * Return the limiter associated with this quota without updating internal last query stats - * @return the quota limiter - */ - synchronized QuotaLimiter getGlobalLimiterWithoutUpdatingLastQuery() { - return globalLimiter; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 3ef704b666b3..6b38635eccc0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -39,10 +39,11 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -329,59 +330,56 @@ private static void deleteQuotas(final Connection connection, final byte[] rowKe } public static Map fetchUserQuotas(final Connection connection, - final List gets, Map tableMachineQuotaFactors, double factor) - throws IOException { - long nowTs = EnvironmentEdgeManager.currentTime(); - Result[] results = doGet(connection, gets); - - Map userQuotas = new HashMap<>(results.length); - for (int i = 0; i < results.length; ++i) { - byte[] key = gets.get(i).getRow(); - assert isUserRowKey(key); - String user = getUserFromRowKey(key); - - if (results[i].isEmpty()) { - userQuotas.put(user, buildDefaultUserQuotaState(connection.getConfiguration(), nowTs)); - continue; - } - - final UserQuotaState quotaInfo = new UserQuotaState(nowTs); - userQuotas.put(user, quotaInfo); - - assert Bytes.equals(key, results[i].getRow()); - - try { - parseUserResult(user, results[i], new UserQuotasVisitor() { - @Override - public void visitUserQuotas(String userName, String namespace, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(namespace, quotas); + Map tableMachineQuotaFactors, double factor) throws IOException { + Map userQuotas = new HashMap<>(); + try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_USER_ROW_KEY_PREFIX); + try (ResultScanner resultScanner = table.getScanner(scan)) { + for (Result result : resultScanner) { + byte[] key = result.getRow(); + assert isUserRowKey(key); + String user = getUserFromRowKey(key); + + final UserQuotaState quotaInfo = new UserQuotaState(); + userQuotas.put(user, quotaInfo); + + try { + parseUserResult(user, result, new UserQuotasVisitor() { + @Override + public void visitUserQuotas(String userName, String namespace, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, factor); + quotaInfo.setQuotas(namespace, quotas); + } + + @Override + public void visitUserQuotas(String userName, TableName table, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, + tableMachineQuotaFactors.containsKey(table) + ? tableMachineQuotaFactors.get(table) + : 1); + quotaInfo.setQuotas(table, quotas); + } + + @Override + public void visitUserQuotas(String userName, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, factor); + quotaInfo.setQuotas(quotas); + } + }); + } catch (IOException e) { + LOG.error("Unable to parse user '" + user + "' quotas", e); + userQuotas.remove(user); } - - @Override - public void visitUserQuotas(String userName, TableName table, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, - tableMachineQuotaFactors.containsKey(table) - ? tableMachineQuotaFactors.get(table) - : 1); - quotaInfo.setQuotas(table, quotas); - } - - @Override - public void visitUserQuotas(String userName, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(quotas); - } - }); - } catch (IOException e) { - LOG.error("Unable to parse user '" + user + "' quotas", e); - userQuotas.remove(user); + } } } + return userQuotas; } - protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf, long nowTs) { + protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) { QuotaProtos.Throttle.Builder throttleBuilder = QuotaProtos.Throttle.newBuilder(); buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_NUM) @@ -405,7 +403,7 @@ protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf, l buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_REQUEST_HANDLER_USAGE_MS) .ifPresent(throttleBuilder::setReqHandlerUsageMs); - UserQuotaState state = new UserQuotaState(nowTs); + UserQuotaState state = new UserQuotaState(); QuotaProtos.Quotas defaultQuotas = QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build(); state.setQuotas(defaultQuotas); @@ -422,8 +420,11 @@ private static Optional buildDefaultTimedQuota(Configuration conf, S } public static Map fetchTableQuotas(final Connection connection, - final List gets, Map tableMachineFactors) throws IOException { - return fetchGlobalQuotas("table", connection, gets, new KeyFromRow() { + Map tableMachineFactors) throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_TABLE_ROW_KEY_PREFIX); + return fetchGlobalQuotas("table", scan, connection, new KeyFromRow() { @Override public TableName getKeyFromRow(final byte[] row) { assert isTableRowKey(row); @@ -438,8 +439,11 @@ public double getFactor(TableName tableName) { } public static Map fetchNamespaceQuotas(final Connection connection, - final List gets, double factor) throws IOException { - return fetchGlobalQuotas("namespace", connection, gets, new KeyFromRow() { + double factor) throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_NAMESPACE_ROW_KEY_PREFIX); + return fetchGlobalQuotas("namespace", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isNamespaceRowKey(row); @@ -453,9 +457,12 @@ public double getFactor(String s) { }); } - public static Map fetchRegionServerQuotas(final Connection connection, - final List gets) throws IOException { - return fetchGlobalQuotas("regionServer", connection, gets, new KeyFromRow() { + public static Map fetchRegionServerQuotas(final Connection connection) + throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_REGION_SERVER_ROW_KEY_PREFIX); + return fetchGlobalQuotas("regionServer", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isRegionServerRowKey(row); @@ -469,32 +476,34 @@ public double getFactor(String s) { }); } - public static Map fetchGlobalQuotas(final String type, - final Connection connection, final List gets, final KeyFromRow kfr) throws IOException { - long nowTs = EnvironmentEdgeManager.currentTime(); - Result[] results = doGet(connection, gets); + public static Map fetchGlobalQuotas(final String type, final Scan scan, + final Connection connection, final KeyFromRow kfr) throws IOException { - Map globalQuotas = new HashMap<>(results.length); - for (int i = 0; i < results.length; ++i) { - byte[] row = gets.get(i).getRow(); - K key = kfr.getKeyFromRow(row); + Map globalQuotas = new HashMap<>(); + try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { + try (ResultScanner resultScanner = table.getScanner(scan)) { + for (Result result : resultScanner) { - QuotaState quotaInfo = new QuotaState(nowTs); - globalQuotas.put(key, quotaInfo); + byte[] row = result.getRow(); + K key = kfr.getKeyFromRow(row); - if (results[i].isEmpty()) continue; - assert Bytes.equals(row, results[i].getRow()); + QuotaState quotaInfo = new QuotaState(); + globalQuotas.put(key, quotaInfo); - byte[] data = results[i].getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); - if (data == null) continue; + byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); + if (data == null) { + continue; + } - try { - Quotas quotas = quotasFromData(data); - quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); - quotaInfo.setQuotas(quotas); - } catch (IOException e) { - LOG.error("Unable to parse " + type + " '" + key + "' quotas", e); - globalQuotas.remove(key); + try { + Quotas quotas = quotasFromData(data); + quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); + quotaInfo.setQuotas(quotas); + } catch (IOException e) { + LOG.error("Unable to parse {} '{}' quotas", type, key, e); + globalQuotas.remove(key); + } + } } } return globalQuotas; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java index a3ec97994363..877ad195c716 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -42,24 +41,18 @@ public class UserQuotaState extends QuotaState { private Map tableLimiters = null; private boolean bypassGlobals = false; - public UserQuotaState() { - super(); - } - - public UserQuotaState(final long updateTs) { - super(updateTs); - } - @Override public synchronized String toString() { StringBuilder builder = new StringBuilder(); - builder.append("UserQuotaState(ts=" + getLastUpdate()); - if (bypassGlobals) builder.append(" bypass-globals"); + builder.append("UserQuotaState("); + if (bypassGlobals) { + builder.append("bypass-globals"); + } if (isBypass()) { builder.append(" bypass"); } else { - if (getGlobalLimiterWithoutUpdatingLastQuery() != NoopQuotaLimiter.get()) { + if (getGlobalLimiter() != NoopQuotaLimiter.get()) { builder.append(" global-limiter"); } @@ -86,7 +79,7 @@ public synchronized String toString() { /** Returns true if there is no quota information associated to this object */ @Override public synchronized boolean isBypass() { - return !bypassGlobals && getGlobalLimiterWithoutUpdatingLastQuery() == NoopQuotaLimiter.get() + return !bypassGlobals && getGlobalLimiter() == NoopQuotaLimiter.get() && (tableLimiters == null || tableLimiters.isEmpty()) && (namespaceLimiters == null || namespaceLimiters.isEmpty()); } @@ -191,7 +184,6 @@ private static Map updateLimiters(final Map userQuotaState.getLastUpdate() != 0); - long lastUpdate = userQuotaState.getLastUpdate(); - - // refresh should not apply to recently refreshed quota - quotaCache.triggerCacheRefresh(); - Thread.sleep(250); - long newLastUpdate = userQuotaState.getLastUpdate(); - assertEquals(lastUpdate, newLastUpdate); - - quotaCache.triggerCacheRefresh(); - waitMinuteQuota(); - // should refresh after time has passed - TEST_UTIL.waitFor(5_000, () -> lastUpdate != userQuotaState.getLastUpdate()); - } - @Test public void testUserQuotaLookup() throws Exception { QuotaCache quotaCache = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java new file mode 100644 index 000000000000..2c33b265771a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; + +/** + * Tests of QuotaCache that don't require a minicluster, unlike in TestQuotaCache + */ +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestQuotaCache2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestQuotaCache2.class); + + @Test + public void testPreserveLimiterAvailability() throws Exception { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + oldState.setGlobalLimiter(limiter1); + + // consume one byte from the limiter, so 99 will be left + limiter1.consumeRead(1, 1, false); + + // establish new cache, also with a limiter for 100 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 99 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(99, updatedLimiter.getReadAvailable()); + } + + @Test + public void testClobberLimiterLimit() throws Exception { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + oldState.setGlobalLimiter(limiter1); + + // establish new cache, also with a limiter for 100 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(50).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 99 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(50, updatedLimiter.getReadLimit()); + } + + @Test + public void testForgetsDeletedQuota() { + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table1", oldState); + + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table2", newState); + + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + assertTrue(newCache.containsKey("my_table2")); + assertFalse(newCache.containsKey("my_table1")); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java index 59b26f3f0d91..ff4b6bc9949b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -81,67 +80,38 @@ public void testSimpleQuotaStateOperation() { assertThrottleException(quotaInfo.getTableLimiter(tableName), NUM_TABLE_THROTTLE); } - @Test - public void testQuotaStateUpdateBypassThrottle() { - final long LAST_UPDATE = 10; - - UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); - assertTrue(quotaInfo.isBypass()); - - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE); - assertEquals(LAST_UPDATE, otherQuotaState.getLastUpdate()); - assertTrue(otherQuotaState.isBypass()); - - quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE, quotaInfo.getLastUpdate()); - assertTrue(quotaInfo.isBypass()); - assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); - assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); - } - @Test public void testQuotaStateUpdateGlobalThrottle() { final int NUM_GLOBAL_THROTTLE_1 = 3; final int NUM_GLOBAL_THROTTLE_2 = 11; - final long LAST_UPDATE_1 = 10; - final long LAST_UPDATE_2 = 20; - final long LAST_UPDATE_3 = 30; QuotaState quotaInfo = new QuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add global throttle - QuotaState otherQuotaState = new QuotaState(LAST_UPDATE_1); + QuotaState otherQuotaState = new QuotaState(); otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_1); // Update global Throttle - otherQuotaState = new QuotaState(LAST_UPDATE_2); + otherQuotaState = new QuotaState(); otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); - assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_2 - NUM_GLOBAL_THROTTLE_1); // Remove global throttle - otherQuotaState = new QuotaState(LAST_UPDATE_3); - assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + otherQuotaState = new QuotaState(); assertTrue(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); assertNoopLimiter(quotaInfo.getGlobalLimiter()); } @@ -155,37 +125,29 @@ public void testQuotaStateUpdateTableThrottle() { final int TABLE_A_THROTTLE_2 = 11; final int TABLE_B_THROTTLE = 4; final int TABLE_C_THROTTLE = 5; - final long LAST_UPDATE_1 = 10; - final long LAST_UPDATE_2 = 20; - final long LAST_UPDATE_3 = 30; UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A B table limiters - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + UserQuotaState otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); otherQuotaState.setQuotas(tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getTableLimiter(tableNameA), TABLE_A_THROTTLE_1); assertThrottleException(quotaInfo.getTableLimiter(tableNameB), TABLE_B_THROTTLE); assertNoopLimiter(quotaInfo.getTableLimiter(tableNameC)); // Add C, Remove B, Update A table limiters - otherQuotaState = new UserQuotaState(LAST_UPDATE_2); + otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); otherQuotaState.setQuotas(tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); - assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getTableLimiter(tableNameA), TABLE_A_THROTTLE_2 - TABLE_A_THROTTLE_1); @@ -193,12 +155,10 @@ public void testQuotaStateUpdateTableThrottle() { assertNoopLimiter(quotaInfo.getTableLimiter(tableNameB)); // Remove table limiters - otherQuotaState = new UserQuotaState(LAST_UPDATE_3); - assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + otherQuotaState = new UserQuotaState(); assertTrue(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); } @@ -207,20 +167,16 @@ public void testQuotaStateUpdateTableThrottle() { public void testTableThrottleWithBatch() { final TableName TABLE_A = TableName.valueOf("TableA"); final int TABLE_A_THROTTLE_1 = 3; - final long LAST_UPDATE_1 = 10; UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A table limiters - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + UserQuotaState otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); QuotaLimiter limiter = quotaInfo.getTableLimiter(TABLE_A); try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java index 5ae9de1fbf16..66996f366610 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java @@ -88,7 +88,6 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); TEST_UTIL.startMiniCluster(1); TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; tables = new Table[TABLE_NAMES.length]; for (int i = 0; i < TABLE_NAMES.length; ++i) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java index 683d189b761b..7917f3c0847f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java @@ -65,7 +65,6 @@ public static void setUpBeforeClass() throws Exception { CUSTOM_OVERRIDE_KEY); TEST_UTIL.startMiniCluster(NUM_SERVERS); TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; TEST_UTIL.createTable(TABLE_NAME, FAMILY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java index 8a9863132e81..58b15ec24294 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -74,7 +75,6 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.createTable(TABLE_NAME, FAMILY); TEST_UTIL.waitTableAvailable(TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; TEST_UTIL.flush(TABLE_NAME); } @@ -104,11 +104,12 @@ public void testHandlerUsageThrottleForWrites() throws Exception { } } - private void configureThrottle() throws IOException { + private void configureThrottle() throws Exception { try (Admin admin = TEST_UTIL.getAdmin()) { admin.setQuota(QuotaSettingsFactory.throttleUser(getUserName(), - ThrottleType.REQUEST_HANDLER_USAGE_MS, 10000, TimeUnit.SECONDS)); + ThrottleType.REQUEST_HANDLER_USAGE_MS, 1, TimeUnit.SECONDS)); } + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); } private void unthrottleUser() throws Exception { @@ -116,6 +117,7 @@ private void unthrottleUser() throws Exception { admin.setQuota(QuotaSettingsFactory.unthrottleUserByThrottleType(getUserName(), ThrottleType.REQUEST_HANDLER_USAGE_MS)); } + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); } private static String getUserName() throws IOException { From 8831bb61a7e7681a49fe1fd45e07af3af793a2f6 Mon Sep 17 00:00:00 2001 From: Ruan Hui Date: Wed, 6 Jul 2022 10:59:13 +0800 Subject: [PATCH 059/336] HBASE-27157 Potential race condition in WorkerAssigner (#4577) Close #7299 Co-authored-by: Duo Zhang Signed-off-by: Duo Zhang Signed-off-by: Lijin Bin (cherry picked from commit 0d1ff8aa9bc21b73f2cf624d35fdcea1417de613) --- .../hadoop/hbase/master/SplitWALManager.java | 18 +-- .../hadoop/hbase/master/WorkerAssigner.java | 33 ++--- .../procedure/SnapshotVerifyProcedure.java | 3 +- .../master/procedure/SplitWALProcedure.java | 2 +- .../master/snapshot/SnapshotManager.java | 16 +-- .../hbase/master/TestSplitWALManager.java | 136 +++++++++--------- 6 files changed, 100 insertions(+), 108 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java index 18dfc7d493bf..32b2f4d21f29 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java @@ -26,7 +26,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -35,7 +34,6 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; import org.apache.hadoop.hbase.master.procedure.SplitWALProcedure; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; @@ -153,25 +151,19 @@ List createSplitWALProcedures(List splittingWALs, */ public ServerName acquireSplitWALWorker(Procedure procedure) throws ProcedureSuspendedException { - Optional worker = splitWorkerAssigner.acquire(); - if (worker.isPresent()) { - LOG.debug("Acquired split WAL worker={}", worker.get()); - return worker.get(); - } - splitWorkerAssigner.suspend(procedure); - throw new ProcedureSuspendedException(); + ServerName worker = splitWorkerAssigner.acquire(procedure); + LOG.debug("Acquired split WAL worker={}", worker); + return worker; } /** * After the worker finished the split WAL task, it will release the worker, and wake up all the * suspend procedures in the ProcedureEvent - * @param worker worker which is about to release - * @param scheduler scheduler which is to wake up the procedure event + * @param worker worker which is about to release */ - public void releaseSplitWALWorker(ServerName worker, MasterProcedureScheduler scheduler) { + public void releaseSplitWALWorker(ServerName worker) { LOG.debug("Release split WAL worker={}", worker); splitWorkerAssigner.release(worker); - splitWorkerAssigner.wake(scheduler); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/WorkerAssigner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/WorkerAssigner.java index b6df41acee23..7b1ec80cab4a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/WorkerAssigner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/WorkerAssigner.java @@ -23,9 +23,9 @@ import java.util.Map; import java.util.Optional; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureEvent; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; import org.apache.yetus.audience.InterfaceAudience; /** @@ -51,36 +51,37 @@ public WorkerAssigner(MasterServices master, int maxTasks, ProcedureEvent eve } } - public synchronized Optional acquire() { + public synchronized ServerName acquire(Procedure proc) throws ProcedureSuspendedException { List serverList = master.getServerManager().getOnlineServersList(); Collections.shuffle(serverList); Optional worker = serverList.stream() .filter( serverName -> !currentWorkers.containsKey(serverName) || currentWorkers.get(serverName) > 0) .findAny(); - worker.ifPresent(name -> currentWorkers.compute(name, (serverName, - availableWorker) -> availableWorker == null ? maxTasks - 1 : availableWorker - 1)); - return worker; + if (worker.isPresent()) { + ServerName sn = worker.get(); + currentWorkers.compute(sn, (serverName, + availableWorker) -> availableWorker == null ? maxTasks - 1 : availableWorker - 1); + return sn; + } else { + event.suspend(); + event.suspendIfNotReady(proc); + throw new ProcedureSuspendedException(); + } } public synchronized void release(ServerName serverName) { currentWorkers.compute(serverName, (k, v) -> v == null ? null : v + 1); - } - - public void suspend(Procedure proc) { - event.suspend(); - event.suspendIfNotReady(proc); - } - - public void wake(MasterProcedureScheduler scheduler) { if (!event.isReady()) { - event.wake(scheduler); + event.wake(master.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler()); } } @Override - public void serverAdded(ServerName worker) { - this.wake(master.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler()); + public synchronized void serverAdded(ServerName worker) { + if (!event.isReady()) { + event.wake(master.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler()); + } } public synchronized void addUsedWorker(ServerName worker) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotVerifyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotVerifyProcedure.java index a3e126484c34..34a12ed52b1a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotVerifyProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotVerifyProcedure.java @@ -109,8 +109,7 @@ protected synchronized boolean complete(MasterProcedureEnv env, Throwable error) setFailure("verify-snapshot", e); } finally { // release the worker - env.getMasterServices().getSnapshotManager().releaseSnapshotVerifyWorker(this, targetServer, - env.getProcedureScheduler()); + env.getMasterServices().getSnapshotManager().releaseSnapshotVerifyWorker(this, targetServer); } return isProcedureCompleted; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java index 699834f9c1d7..98c2c0ec6930 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java @@ -90,7 +90,7 @@ protected Flow executeFromState(MasterProcedureEnv env, MasterProcedureProtos.Sp skipPersistence(); throw new ProcedureSuspendedException(); } - splitWALManager.releaseSplitWALWorker(worker, env.getProcedureScheduler()); + splitWALManager.releaseSplitWALWorker(worker); if (!finished) { LOG.warn("Failed to split wal {} by server {}, retry...", walPath, worker); setNextState(MasterProcedureProtos.SplitWALState.ACQUIRE_SPLIT_WAL_WORKER); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 8936fbfc7fa7..bb64062cf1bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -26,7 +26,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -65,7 +64,6 @@ import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; import org.apache.hadoop.hbase.master.procedure.CloneSnapshotProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; import org.apache.hadoop.hbase.master.procedure.RestoreSnapshotProcedure; import org.apache.hadoop.hbase.master.procedure.SnapshotProcedure; @@ -1470,20 +1468,14 @@ public boolean snapshotProcedureEnabled() { public ServerName acquireSnapshotVerifyWorker(SnapshotVerifyProcedure procedure) throws ProcedureSuspendedException { - Optional worker = verifyWorkerAssigner.acquire(); - if (worker.isPresent()) { - LOG.debug("{} Acquired verify snapshot worker={}", procedure, worker.get()); - return worker.get(); - } - verifyWorkerAssigner.suspend(procedure); - throw new ProcedureSuspendedException(); + ServerName worker = verifyWorkerAssigner.acquire(procedure); + LOG.debug("{} Acquired verify snapshot worker={}", procedure, worker); + return worker; } - public void releaseSnapshotVerifyWorker(SnapshotVerifyProcedure procedure, ServerName worker, - MasterProcedureScheduler scheduler) { + public void releaseSnapshotVerifyWorker(SnapshotVerifyProcedure procedure, ServerName worker) { LOG.debug("{} Release verify snapshot worker={}", procedure, worker); verifyWorkerAssigner.release(worker); - verifyWorkerAssigner.wake(scheduler); } private void restoreWorkers() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index 8818609c7310..40af3f2f9015 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hbase.master; -import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK; -import static org.apache.hadoop.hbase.HConstants.HBASE_SPLIT_WAL_MAX_SPLITTER; import static org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface.ServerOperationType.SPLIT_WAL; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -33,6 +36,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -46,10 +50,10 @@ import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; @@ -63,7 +67,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; @Category({ MasterTests.class, LargeTests.class }) - public class TestSplitWALManager { @ClassRule @@ -78,10 +81,11 @@ public class TestSplitWALManager { private byte[] FAMILY; @Before - public void setup() throws Exception { + public void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); - TEST_UTIL.getConfiguration().setBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, false); - TEST_UTIL.getConfiguration().setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, 1); + TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK, false); + TEST_UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 5); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_SPLIT_WAL_MAX_SPLITTER, 1); TEST_UTIL.startMiniCluster(3); master = TEST_UTIL.getHBaseCluster().getMaster(); splitWALManager = master.getSplitWALManager(); @@ -90,7 +94,7 @@ public void setup() throws Exception { } @After - public void teardown() throws Exception { + public void tearDown() throws Exception { TEST_UTIL.shutdownMiniCluster(); } @@ -98,57 +102,61 @@ public void teardown() throws Exception { public void testAcquireAndRelease() throws Exception { List testProcedures = new ArrayList<>(); for (int i = 0; i < 4; i++) { - testProcedures - .add(new FakeServerProcedure(TEST_UTIL.getHBaseCluster().getServerHoldingMeta())); + testProcedures.add(new FakeServerProcedure( + ServerName.valueOf("server" + i, 12345, EnvironmentEdgeManager.currentTime()))); } - ServerName server = splitWALManager.acquireSplitWALWorker(testProcedures.get(0)); - Assert.assertNotNull(server); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(1))); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(2))); - - Exception e = null; - try { - splitWALManager.acquireSplitWALWorker(testProcedures.get(3)); - } catch (ProcedureSuspendedException suspendException) { - e = suspendException; + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + procExec.submitProcedure(testProcedures.get(0)); + TEST_UTIL.waitFor(10000, () -> testProcedures.get(0).isWorkerAcquired()); + procExec.submitProcedure(testProcedures.get(1)); + procExec.submitProcedure(testProcedures.get(2)); + TEST_UTIL.waitFor(10000, + () -> testProcedures.get(1).isWorkerAcquired() && testProcedures.get(2).isWorkerAcquired()); + + // should get a ProcedureSuspendedException, so it will try to acquire but can not get a worker + procExec.submitProcedure(testProcedures.get(3)); + TEST_UTIL.waitFor(10000, () -> testProcedures.get(3).isTriedToAcquire()); + for (int i = 0; i < 3; i++) { + Thread.sleep(1000); + assertFalse(testProcedures.get(3).isWorkerAcquired()); } - Assert.assertNotNull(e); - Assert.assertTrue(e instanceof ProcedureSuspendedException); - splitWALManager.releaseSplitWALWorker(server, TEST_UTIL.getHBaseCluster().getMaster() - .getMasterProcedureExecutor().getEnvironment().getProcedureScheduler()); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(3))); + // release a worker, the last procedure should be able to get a worker + testProcedures.get(0).countDown(); + TEST_UTIL.waitFor(10000, () -> testProcedures.get(3).isWorkerAcquired()); + + for (int i = 1; i < 4; i++) { + testProcedures.get(i).countDown(); + } + for (int i = 0; i < 4; i++) { + final int index = i; + TEST_UTIL.waitFor(10000, () -> testProcedures.get(index).isFinished()); + } } @Test public void testAddNewServer() throws Exception { List testProcedures = new ArrayList<>(); for (int i = 0; i < 4; i++) { - testProcedures - .add(new FakeServerProcedure(TEST_UTIL.getHBaseCluster().getServerHoldingMeta())); + testProcedures.add( + new FakeServerProcedure(TEST_UTIL.getHBaseCluster().getRegionServer(1).getServerName())); } ServerName server = splitWALManager.acquireSplitWALWorker(testProcedures.get(0)); - Assert.assertNotNull(server); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(1))); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(2))); - - Exception e = null; - try { - splitWALManager.acquireSplitWALWorker(testProcedures.get(3)); - } catch (ProcedureSuspendedException suspendException) { - e = suspendException; - } - Assert.assertNotNull(e); - Assert.assertTrue(e instanceof ProcedureSuspendedException); + assertNotNull(server); + assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(1))); + assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(2))); + + assertThrows(ProcedureSuspendedException.class, + () -> splitWALManager.acquireSplitWALWorker(testProcedures.get(3))); JVMClusterUtil.RegionServerThread newServer = TEST_UTIL.getHBaseCluster().startRegionServer(); newServer.waitForServerOnline(); - Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(3))); + assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(3))); } @Test public void testCreateSplitWALProcedures() throws Exception { - TEST_UTIL.createTable(TABLE_NAME, FAMILY, TEST_UTIL.KEYS_FOR_HBA_CREATE_TABLE); + TEST_UTIL.createTable(TABLE_NAME, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); // load table TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(TABLE_NAME), FAMILY); ProcedureExecutor masterPE = master.getMasterProcedureExecutor(); @@ -158,21 +166,21 @@ public void testCreateSplitWALProcedures() throws Exception { // Test splitting meta wal FileStatus[] wals = TEST_UTIL.getTestFileSystem().listStatus(metaWALDir, MasterWalManager.META_FILTER); - Assert.assertEquals(1, wals.length); + assertEquals(1, wals.length); List testProcedures = splitWALManager.createSplitWALProcedures(Lists.newArrayList(wals[0]), metaServer); - Assert.assertEquals(1, testProcedures.size()); + assertEquals(1, testProcedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, testProcedures.get(0)); - Assert.assertFalse(TEST_UTIL.getTestFileSystem().exists(wals[0].getPath())); + assertFalse(TEST_UTIL.getTestFileSystem().exists(wals[0].getPath())); // Test splitting wal wals = TEST_UTIL.getTestFileSystem().listStatus(metaWALDir, MasterWalManager.NON_META_FILTER); - Assert.assertEquals(1, wals.length); + assertEquals(1, wals.length); testProcedures = splitWALManager.createSplitWALProcedures(Lists.newArrayList(wals[0]), metaServer); - Assert.assertEquals(1, testProcedures.size()); + assertEquals(1, testProcedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, testProcedures.get(0)); - Assert.assertFalse(TEST_UTIL.getTestFileSystem().exists(wals[0].getPath())); + assertFalse(TEST_UTIL.getTestFileSystem().exists(wals[0].getPath())); } @Test @@ -192,11 +200,11 @@ public void testAcquireAndReleaseSplitWALWorker() throws Exception { ProcedureTestingUtility.submitProcedure(masterPE, failedProcedure, HConstants.NO_NONCE, HConstants.NO_NONCE); TEST_UTIL.waitFor(20000, () -> failedProcedure.isTriedToAcquire()); - Assert.assertFalse(failedProcedure.isWorkerAcquired()); + assertFalse(failedProcedure.isWorkerAcquired()); // let one procedure finish and release worker testProcedures.get(0).countDown(); TEST_UTIL.waitFor(10000, () -> failedProcedure.isWorkerAcquired()); - Assert.assertTrue(testProcedures.get(0).isSuccess()); + assertTrue(testProcedures.get(0).isSuccess()); } @Test @@ -206,14 +214,14 @@ public void testGetWALsToSplit() throws Exception { TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(TABLE_NAME), FAMILY); ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta(); List metaWals = splitWALManager.getWALsToSplit(metaServer, true); - Assert.assertEquals(1, metaWals.size()); + assertEquals(1, metaWals.size()); List wals = splitWALManager.getWALsToSplit(metaServer, false); - Assert.assertEquals(1, wals.size()); + assertEquals(1, wals.size()); ServerName testServer = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream() .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny() .get(); metaWals = splitWALManager.getWALsToSplit(testServer, true); - Assert.assertEquals(0, metaWals.size()); + assertEquals(0, metaWals.size()); } private void splitLogsTestHelper(HBaseTestingUtility testUtil) throws Exception { @@ -233,9 +241,9 @@ private void splitLogsTestHelper(HBaseTestingUtility testUtil) throws Exception .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny() .get(); List procedures = splitWALManager.splitWALs(testServer, false); - Assert.assertEquals(1, procedures.size()); + assertEquals(1, procedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0)); - Assert.assertEquals(0, splitWALManager.getWALsToSplit(testServer, false).size()); + assertEquals(0, splitWALManager.getWALsToSplit(testServer, false).size()); // Validate the old WAL file archive dir Path walRootDir = hmaster.getMasterFileSystem().getWALRootDir(); @@ -244,12 +252,12 @@ private void splitLogsTestHelper(HBaseTestingUtility testUtil) throws Exception int archiveFileCount = walFS.listStatus(walArchivePath).length; procedures = splitWALManager.splitWALs(metaServer, true); - Assert.assertEquals(1, procedures.size()); + assertEquals(1, procedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0)); - Assert.assertEquals(0, splitWALManager.getWALsToSplit(metaServer, true).size()); - Assert.assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size()); + assertEquals(0, splitWALManager.getWALsToSplit(metaServer, true).size()); + assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size()); // There should be archiveFileCount + 1 WALs after SplitWALProcedure finish - Assert.assertEquals("Splitted WAL files should be archived", archiveFileCount + 1, + assertEquals("Splitted WAL files should be archived", archiveFileCount + 1, walFS.listStatus(walArchivePath).length); } @@ -261,8 +269,8 @@ public void testSplitLogs() throws Exception { @Test public void testSplitLogsWithDifferentWalAndRootFS() throws Exception { HBaseTestingUtility testUtil2 = new HBaseTestingUtility(); - testUtil2.getConfiguration().setBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, false); - testUtil2.getConfiguration().setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, 1); + testUtil2.getConfiguration().setBoolean(HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK, false); + testUtil2.getConfiguration().setInt(HConstants.HBASE_SPLIT_WAL_MAX_SPLITTER, 1); Path dir = TEST_UTIL.getDataTestDirOnTestFS("testWalDir"); testUtil2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); CommonFSUtils.setWALRootDir(testUtil2.getConfiguration(), dir); @@ -295,7 +303,7 @@ public void testWorkerReloadWhenMasterRestart() throws Exception { ProcedureTestingUtility.submitProcedure(master.getMasterProcedureExecutor(), failedProcedure, HConstants.NO_NONCE, HConstants.NO_NONCE); TEST_UTIL.waitFor(20000, () -> failedProcedure.isTriedToAcquire()); - Assert.assertFalse(failedProcedure.isWorkerAcquired()); + assertFalse(failedProcedure.isWorkerAcquired()); for (int i = 0; i < 3; i++) { testProcedures.get(i).countDown(); } @@ -307,9 +315,9 @@ public static final class FakeServerProcedure implements ServerProcedureInterface { private ServerName serverName; - private ServerName worker; + private volatile ServerName worker; private CountDownLatch barrier = new CountDownLatch(1); - private boolean triedToAcquire = false; + private volatile boolean triedToAcquire = false; public FakeServerProcedure() { } @@ -348,7 +356,7 @@ protected Flow executeFromState(MasterProcedureEnv env, setNextState(MasterProcedureProtos.SplitWALState.RELEASE_SPLIT_WORKER); return Flow.HAS_MORE_STATE; case RELEASE_SPLIT_WORKER: - splitWALManager.releaseSplitWALWorker(worker, env.getProcedureScheduler()); + splitWALManager.releaseSplitWALWorker(worker); return Flow.NO_MORE_STATE; default: throw new UnsupportedOperationException("unhandled state=" + state); From 162b232669d617e8ce4ce5e35a0a3c9296d9b170 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 15 Sep 2025 18:39:15 +0900 Subject: [PATCH 060/336] HBASE-29577 Fix NPE from RegionServerRpcQuotaManager when reloading configuration (#7285) Signed-off-by: Wellington Chevreuil Signed-off-by: Charles Connell --- .../hadoop/hbase/quotas/RegionServerRpcQuotaManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 958793dcdf00..7a42d0f1aa31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -91,7 +91,9 @@ public void stop() { } public void reload() { - quotaCache.forceSynchronousCacheRefresh(); + if (isQuotaEnabled()) { + quotaCache.forceSynchronousCacheRefresh(); + } } @Override From cef0dc4a63cb55f668265625a8d72ebccdc23ee6 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Tue, 16 Sep 2025 16:55:02 +0200 Subject: [PATCH 061/336] HBASE-29590 Use hadoop 3.4.2 as default hadooop3 dependency (#7302) Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang (cherry picked from commit 0fe3f8a75e48e25e0e0d63e8c66159156e2dce1f) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0910fdc6083d..c67630aa7c6e 100644 --- a/pom.xml +++ b/pom.xml @@ -548,7 +548,7 @@ ${compileSource} 2.10.2 - 3.4.1 + 3.4.2 From f408dc69a3501b158c302a2229e4179330c41705 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 08:14:33 +0200 Subject: [PATCH 062/336] HBASE-29548 Update ApacheDS to 2.0.0.AM27 and ldap-api to 2.1.7 (#7312) Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang (cherry picked from commit dce4acc5f274e6ef05b7e28cf565553c173eac4c) --- hbase-http/pom.xml | 4 ++ .../hadoop/hbase/http/LdapServerTestBase.java | 61 +++++++++++++++---- .../hadoop/hbase/http/TestLdapAdminACL.java | 23 +++---- .../hadoop/hbase/http/TestLdapHttpServer.java | 20 +++--- pom.xml | 4 +- 5 files changed, 74 insertions(+), 38 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 7dfdd25dc64f..bcbcb4d5cbaf 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -486,6 +486,10 @@ org.bouncycastle bcprov-jdk15on + + org.bouncycastle + bcpkix-jdk15on + diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java index bbf35b8585f6..8856aaa0e205 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/LdapServerTestBase.java @@ -21,34 +21,73 @@ import java.net.HttpURLConnection; import java.net.URL; import org.apache.commons.codec.binary.Base64; -import org.apache.directory.server.core.integ.CreateLdapServerRule; +import org.apache.directory.ldap.client.template.LdapConnectionTemplate; +import org.apache.directory.server.core.api.DirectoryService; +import org.apache.directory.server.core.integ.ApacheDSTestExtension; +import org.apache.directory.server.ldap.LdapServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.http.resource.JerseyResource; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Base class for setting up and testing an HTTP server with LDAP authentication. */ +@ExtendWith(ApacheDSTestExtension.class) public class LdapServerTestBase extends HttpServerFunctionalTest { private static final Logger LOG = LoggerFactory.getLogger(LdapServerTestBase.class); - @ClassRule - public static CreateLdapServerRule ldapRule = new CreateLdapServerRule(); - protected static HttpServer server; protected static URL baseUrl; + /** + * The following fields are set by ApacheDSTestExtension. These are normally inherited from + * AbstractLdapTestUnit, but this class already has a parent. We only use ldapServer, but + * declaring that one alone does not work. + */ + + /** The class DirectoryService instance */ + public static DirectoryService classDirectoryService; + + /** The test DirectoryService instance */ + public static DirectoryService methodDirectoryService; + + /** The current DirectoryService instance */ + public static DirectoryService directoryService; + + /** The class LdapServer instance */ + public static LdapServer classLdapServer; + + /** The test LdapServer instance */ + public static LdapServer methodLdapServer; + + /** The current LdapServer instance */ + public static LdapServer ldapServer; + + /** The Ldap connection template */ + public static LdapConnectionTemplate ldapConnectionTemplate; + + /** The current revision */ + public static long revision = 0L; + + /** + * End of fields required by ApacheDSTestExtension + */ + private static final String AUTH_TYPE = "Basic "; + protected static LdapServer getLdapServer() { + return classLdapServer; + } + /** * Sets up the HTTP server with LDAP authentication before any tests are run. * @throws Exception if an error occurs during server setup */ - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); setLdapConfigurations(conf); @@ -66,7 +105,7 @@ public static void setupServer() throws Exception { * Stops the HTTP server after all tests are completed. * @throws Exception if an error occurs during server shutdown */ - @AfterClass + @AfterAll public static void stopServer() throws Exception { try { if (null != server) { @@ -90,8 +129,8 @@ protected static void setLdapConfigurations(Configuration conf) { conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY, "org.apache.hadoop.hbase.http.lib.AuthenticationFilterInitializer"); conf.set("hadoop.http.authentication.type", "ldap"); - conf.set("hadoop.http.authentication.ldap.providerurl", String.format("ldap://%s:%s", - LdapConstants.LDAP_SERVER_ADDR, ldapRule.getLdapServer().getPort())); + conf.set("hadoop.http.authentication.ldap.providerurl", + String.format("ldap://%s:%s", LdapConstants.LDAP_SERVER_ADDR, getLdapServer().getPort())); conf.set("hadoop.http.authentication.ldap.enablestarttls", "false"); conf.set("hadoop.http.authentication.ldap.basedn", LdapConstants.LDAP_BASE_DN); } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java index 459865509630..900c1fef07b1 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.HttpURLConnection; +import java.util.concurrent.TimeUnit; import org.apache.directory.server.annotations.CreateLdapServer; import org.apache.directory.server.annotations.CreateTransport; import org.apache.directory.server.core.annotations.ApplyLdifs; @@ -29,21 +30,19 @@ import org.apache.directory.server.core.annotations.CreatePartition; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.http.resource.JerseyResource; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test class for admin ACLs with LDAP authentication on the HttpServer. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag("org.apache.hadoop.hbase.testclassification.MiscTests") +@Tag("org.apache.hadoop.hbase.testclassification.SmallTests") @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapAdminACL", allowAnonAccess = true, @@ -55,18 +54,16 @@ "dn: uid=jdoe," + LdapConstants.LDAP_BASE_DN, "cn: John Doe", "sn: Doe", "objectClass: inetOrgPerson", "uid: jdoe", "userPassword: secure123" }) +@Timeout(value = 1, unit = TimeUnit.MINUTES) public class TestLdapAdminACL extends LdapServerTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLdapAdminACL.class); private static final Logger LOG = LoggerFactory.getLogger(TestLdapAdminACL.class); private static final String ADMIN_CREDENTIALS = "bjones:p@ssw0rd"; private static final String NON_ADMIN_CREDENTIALS = "jdoe:secure123"; private static final String WRONG_CREDENTIALS = "bjones:password"; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { Configuration conf = new Configuration(); setLdapConfigurationWithACLs(conf); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java index bff4dc9d9591..66b3b2924eed 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java @@ -17,27 +17,26 @@ */ package org.apache.hadoop.hbase.http; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.HttpURLConnection; +import java.util.concurrent.TimeUnit; import org.apache.directory.server.annotations.CreateLdapServer; import org.apache.directory.server.annotations.CreateTransport; import org.apache.directory.server.core.annotations.ApplyLdifs; import org.apache.directory.server.core.annotations.ContextEntry; import org.apache.directory.server.core.annotations.CreateDS; import org.apache.directory.server.core.annotations.CreatePartition; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; /** * Test class for LDAP authentication on the HttpServer. */ -@Category({ MiscTests.class, SmallTests.class }) +@Tag("org.apache.hadoop.hbase.testclassification.MiscTests") +@Tag("org.apache.hadoop.hbase.testclassification.SmallTests") @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapHttpServer", allowAnonAccess = true, @@ -46,12 +45,9 @@ + "dc: example\n" + "objectClass: top\n" + "objectClass: domain\n\n")) }) @ApplyLdifs({ "dn: uid=bjones," + LdapConstants.LDAP_BASE_DN, "cn: Bob Jones", "sn: Jones", "objectClass: inetOrgPerson", "uid: bjones", "userPassword: p@ssw0rd" }) +@Timeout(value = 1, unit = TimeUnit.MINUTES) public class TestLdapHttpServer extends LdapServerTestBase { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestLdapHttpServer.class); - private static final String BJONES_CREDENTIALS = "bjones:p@ssw0rd"; private static final String WRONG_CREDENTIALS = "bjones:password"; diff --git a/pom.xml b/pom.xml index c67630aa7c6e..25b4616d8b7a 100644 --- a/pom.xml +++ b/pom.xml @@ -790,8 +790,8 @@ 6.29.0 5.23.0 - 2.0.0.AM26 - 2.0.0 + 2.0.0.AM27 + 2.1.7 From a3e2f231244e2b3dc5ff08f90175293ffcc31298 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 08:42:48 +0200 Subject: [PATCH 063/336] HBASE-29602 Add -Djava.security.manager=allow to JDK18+ surefire JVM flags (#7316) Signed-off-by: Duo Zhang Signed-off-by: Balazs Meszaros (cherry picked from commit 78f333bedefbeab656196919100935742d0fb645) --- pom.xml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pom.xml b/pom.xml index 25b4616d8b7a..59b8cbd097bb 100644 --- a/pom.xml +++ b/pom.xml @@ -765,6 +765,15 @@ --add-opens java.base/sun.security.x509=ALL-UNNAMED --add-opens java.base/sun.security.util=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED + + -Djava.security.manager=allow ${hbase-surefire.argLine} @{jacocoArgLine} 1.5.1 @@ -3108,6 +3117,19 @@ ${hbase-surefire.argLine} @{jacocoArgLine} + + + build-with-jdk18 + + [18,) + + + ${hbase-surefire.jdk11.flags} + ${hbase-surefire.jdk17.flags} + ${hbase-surefire.jdk18.flags} + ${hbase-surefire.argLine} + @{jacocoArgLine} + From 5a0e8a1d0db72a498b78597009c6bacc15a87e53 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 08:57:48 +0200 Subject: [PATCH 064/336] HBASE-29601 Handle Junit 5 tests in TestCheckTestClasses (#7310) Signed-off-by: Duo Zhang (cherry picked from commit df08f0f0be73638520b0358ae8105569d9f825c8) --- .../apache/hadoop/hbase/ClassTestFinder.java | 19 +++++++++++++++++-- .../hadoop/hbase/TestCheckTestClasses.java | 8 ++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java index 1bc648aeb0b5..dc51187e3cf8 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java @@ -19,9 +19,11 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Pattern; -import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; import org.junit.runners.Suite; /** @@ -46,6 +48,16 @@ public static Class[] getCategoryAnnotations(Class c) { return new Class[0]; } + public static String[] getTagAnnotations(Class c) { + // TODO handle optional Tags annotation + Tag[] tags = c.getAnnotationsByType(Tag.class); + List values = new ArrayList<>(); + for (Tag tag : tags) { + values.add(tag.value()); + } + return values.toArray(new String[values.size()]); + } + /** Filters both test classes and anything in the hadoop-compat modules */ public static class TestFileNameFilter implements FileNameFilter, ResourcePathFilter { private static final Pattern hadoopCompactRe = Pattern.compile("hbase-hadoop\\d?-compat"); @@ -92,7 +104,10 @@ private boolean isTestClass(Class c) { } for (Method met : c.getMethods()) { - if (met.getAnnotation(Test.class) != null) { + if ( + met.getAnnotation(org.junit.Test.class) != null + || met.getAnnotation(org.junit.jupiter.api.Test.class) != null + ) { return true; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java index 3d3ca12bd82d..c2b007280b4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java @@ -45,11 +45,15 @@ public void checkClasses() throws Exception { List> badClasses = new java.util.ArrayList<>(); ClassTestFinder classFinder = new ClassTestFinder(); for (Class c : classFinder.findClasses(false)) { - if (ClassTestFinder.getCategoryAnnotations(c).length == 0) { + if ( + ClassTestFinder.getCategoryAnnotations(c).length == 0 + && ClassTestFinder.getTagAnnotations(c).length == 0 + ) { badClasses.add(c); } } - assertTrue("There are " + badClasses.size() + " test classes without category: " + badClasses, + assertTrue( + "There are " + badClasses.size() + " test classes without category and tag: " + badClasses, badClasses.isEmpty()); } } From 6c97ac7dc6129f27489c27971cc5b12c7086ae83 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 10:01:19 +0200 Subject: [PATCH 065/336] HBASE-29602 Add -Djava.security.manager=allow to JDK18+ surefire JVM flags (addendum: run spotless) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 59b8cbd097bb..fe8249af5e5c 100644 --- a/pom.xml +++ b/pom.xml @@ -3118,7 +3118,7 @@ @{jacocoArgLine} - + build-with-jdk18 [18,) From f2a8a5275c620a9b294032044fdff67eb994a7ef Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 10:27:09 +0200 Subject: [PATCH 066/336] HBASE-29592 Add hadoop 3.4.2 in client integration tests (#7306) Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang (cherry picked from commit 40b1ffc51002f3d43c7ffc0556fc8bc650aea0ce) --- dev-support/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 4bc418017c0a..c550272cc3f8 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -59,8 +59,8 @@ pipeline { ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" // These are dependent on the branch - HADOOP3_VERSIONS = "3.3.5,3.3.6,3.4.0,3.4.1" - HADOOP3_DEFAULT_VERSION = "3.4.1" + HADOOP3_VERSIONS = "3.3.5,3.3.6,3.4.0,3.4.1,3.4.2" + HADOOP3_DEFAULT_VERSION = "3.4.2" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. From 6aebeac55f7f5ef3cd7ce3bc6884d2a27953a2ae Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 18 Sep 2025 10:39:08 +0200 Subject: [PATCH 067/336] HBASE-29610 Add and use String constants for Junit 5 @Tag annotations (#7323) Signed-off-by: Duo Zhang (cherry picked from commit 35faadbbef21c2d6c8b46a0685cd8c1b93b0498a) --- .../apache/hadoop/hbase/testclassification/ClientTests.java | 1 + .../hadoop/hbase/testclassification/CoprocessorTests.java | 1 + .../apache/hadoop/hbase/testclassification/FilterTests.java | 1 + .../apache/hadoop/hbase/testclassification/FlakeyTests.java | 1 + .../org/apache/hadoop/hbase/testclassification/IOTests.java | 1 + .../hadoop/hbase/testclassification/IntegrationTests.java | 1 + .../apache/hadoop/hbase/testclassification/LargeTests.java | 1 + .../hadoop/hbase/testclassification/MapReduceTests.java | 1 + .../apache/hadoop/hbase/testclassification/MasterTests.java | 1 + .../apache/hadoop/hbase/testclassification/MediumTests.java | 1 + .../hadoop/hbase/testclassification/MetricsTests.java | 1 + .../apache/hadoop/hbase/testclassification/MiscTests.java | 1 + .../apache/hadoop/hbase/testclassification/RPCTests.java | 1 + .../hadoop/hbase/testclassification/RegionServerTests.java | 1 + .../hadoop/hbase/testclassification/ReplicationTests.java | 1 + .../apache/hadoop/hbase/testclassification/RestTests.java | 1 + .../hadoop/hbase/testclassification/SecurityTests.java | 1 + .../apache/hadoop/hbase/testclassification/SmallTests.java | 1 + .../hbase/testclassification/VerySlowMapReduceTests.java | 2 ++ .../hbase/testclassification/VerySlowRegionServerTests.java | 2 ++ .../org/apache/hadoop/hbase/testclassification/ZKTests.java | 1 + .../java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java | 6 ++++-- .../org/apache/hadoop/hbase/http/TestLdapHttpServer.java | 6 ++++-- 23 files changed, 31 insertions(+), 4 deletions(-) diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java index d9bae8490637..b0e259e1f9e2 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java @@ -36,4 +36,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface ClientTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ClientTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java index a168adec08af..2dc143e944a0 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/CoprocessorTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface CoprocessorTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.CoprocessorTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java index 84f346baaea2..1b45b583c182 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FilterTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface FilterTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.FilterTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java index c23bfa298b36..0cb861979e08 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/FlakeyTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface FlakeyTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.FlakeyTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java index 8eee0e6ae4b9..be55b3829e52 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IOTests.java @@ -36,4 +36,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface IOTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.IOTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java index 4e555b73fedb..0003cd1db511 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/IntegrationTests.java @@ -34,4 +34,5 @@ * @see LargeTests */ public interface IntegrationTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.IntegrationTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java index b47e5bab9a46..3a24764e706a 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/LargeTests.java @@ -33,4 +33,5 @@ * @see IntegrationTests */ public interface LargeTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.LargeTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java index 0e68ab3c0340..ac5b05e30704 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MapReduceTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MapReduceTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MapReduceTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java index 5dcf51b27e59..0ad843493ec1 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MasterTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MasterTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MasterTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java index d1f836ec0049..548f655c774e 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MediumTests.java @@ -32,4 +32,5 @@ * @see IntegrationTests */ public interface MediumTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MediumTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java index 27beaacf963e..c6985d6b95cc 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MetricsTests.java @@ -21,4 +21,5 @@ * Tag a test that covers our metrics handling. */ public interface MetricsTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MetricsTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java index 695042e801bf..b7b7ad4c3f66 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/MiscTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface MiscTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.MiscTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java index 929bd6487edf..71a24d5d5dd6 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RPCTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RPCTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RPCTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java index 3439afa76eba..d79691d6fac6 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RegionServerTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RegionServerTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RegionServerTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java index df606c960c25..74c65a57982d 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ReplicationTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface ReplicationTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ReplicationTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java index a648b4c39e03..9a73fde57e2c 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RestTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface RestTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.RestTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java index a4e55ad3aba0..939c25c05ff4 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SecurityTests.java @@ -35,4 +35,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface SecurityTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.SecurityTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java index 64d2bce381b6..54e16d7ad1ae 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/SmallTests.java @@ -30,4 +30,5 @@ * @see IntegrationTests */ public interface SmallTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.SmallTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java index d1f433b9719d..dac933ec78e4 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowMapReduceTests.java @@ -36,4 +36,6 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface VerySlowMapReduceTests { + public static final String TAG = + "org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java index f556979e5b6a..1583de103e38 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/VerySlowRegionServerTests.java @@ -36,4 +36,6 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface VerySlowRegionServerTests { + public static final String TAG = + "org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests"; } diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java index 9fa0579ed47e..a318b388ef72 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ZKTests.java @@ -22,4 +22,5 @@ * {@code RecoverableZooKeeper}, not for tests which depend on ZooKeeper. */ public interface ZKTests { + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ZKTests"; } diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java index 900c1fef07b1..c4fd208fa7ce 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java @@ -31,6 +31,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hbase.http.resource.JerseyResource; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -41,8 +43,8 @@ /** * Test class for admin ACLs with LDAP authentication on the HttpServer. */ -@Tag("org.apache.hadoop.hbase.testclassification.MiscTests") -@Tag("org.apache.hadoop.hbase.testclassification.SmallTests") +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapAdminACL", allowAnonAccess = true, diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java index 66b3b2924eed..9faa8dc49fb6 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java @@ -28,6 +28,8 @@ import org.apache.directory.server.core.annotations.ContextEntry; import org.apache.directory.server.core.annotations.CreateDS; import org.apache.directory.server.core.annotations.CreatePartition; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -35,8 +37,8 @@ /** * Test class for LDAP authentication on the HttpServer. */ -@Tag("org.apache.hadoop.hbase.testclassification.MiscTests") -@Tag("org.apache.hadoop.hbase.testclassification.SmallTests") +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) @CreateLdapServer( transports = { @CreateTransport(protocol = "LDAP", address = LdapConstants.LDAP_SERVER_ADDR), }) @CreateDS(name = "TestLdapHttpServer", allowAnonAccess = true, From 8fbc35881284de5eb77060082365e7aaf5dce12d Mon Sep 17 00:00:00 2001 From: Sreenivasulu Date: Thu, 18 Sep 2025 14:04:33 +0530 Subject: [PATCH 068/336] HBASE-29587 Set Test category for TestSnapshotProcedureEarlyExpiration (#7292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dávid Paksy (cherry picked from commit 8799c13cd9713660a13b4d34ac9e37a0a59c4191) --- .../procedure/TestSnapshotProcedureEarlyExpiration.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java index e5d21159e506..199fbfe2a63a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureEarlyExpiration.java @@ -34,16 +34,20 @@ import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.RegionSplitter; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +@Category({ MasterTests.class, MediumTests.class }) public class TestSnapshotProcedureEarlyExpiration extends TestSnapshotProcedure { @ClassRule public static final HBaseClassTestRule CLASS_RULE = From 6e679a0a3cd028ee2c5a71b1b6c197f81ee7d2a6 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 18 Sep 2025 17:06:30 +0800 Subject: [PATCH 069/336] HBASE-29591 Add hadoop 3.4.2 in hadoop check (#7320) Signed-off-by: Istvan Toth (cherry picked from commit da7325b77d38f5881679675373dd434d8fa1c013) --- dev-support/hbase-personality.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 46f08276c651..9a5d34cc2138 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -612,17 +612,17 @@ function hadoopcheck_rebuild # TODO remove this on non 2.5 branches ? yetus_info "Setting Hadoop 3 versions to test based on branch-2.5 rules" if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.2.4 3.3.6 3.4.0" + hbase_hadoop3_versions="3.2.4 3.3.6 3.4.1" else - hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6 3.4.0" + hbase_hadoop3_versions="3.2.3 3.2.4 3.3.2 3.3.3 3.3.4 3.3.5 3.3.6 3.4.0 3.4.1" fi else yetus_info "Setting Hadoop 3 versions to test based on branch-2.6+/master/feature branch rules" # Isn't runnung these tests with the default Hadoop version redundant ? if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then - hbase_hadoop3_versions="3.3.6 3.4.0" + hbase_hadoop3_versions="3.3.6 3.4.1" else - hbase_hadoop3_versions="3.3.5 3.3.6 3.4.0" + hbase_hadoop3_versions="3.3.5 3.3.6 3.4.0 3.4.1" fi fi From 81d2d69c881858e825767ca4bd72c26201f7dc62 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 20 Sep 2025 16:18:21 +0800 Subject: [PATCH 070/336] HBASE-29608 Add test to make sure we do not have copy paste errors in the TAG value (#7324) Signed-off-by: Istvan Toth (cherry picked from commit 42fc87d3ae9193c7119b2385d3ba990af56b55de) --- .../hadoop/hbase/TestJUnit5TagConstants.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java new file mode 100644 index 000000000000..43607e171817 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Field; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** + * Verify that the values are all correct. + */ +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +// TODO: this is the timeout for each method, not the whole class +@Timeout(value = 1, unit = TimeUnit.MINUTES) +public class TestJUnit5TagConstants { + + @Test + public void testVerify() throws Exception { + ClassFinder finder = new ClassFinder(getClass().getClassLoader()); + for (Class annoClazz : finder.findClasses(ClientTests.class.getPackage().getName(), false)) { + Field field = annoClazz.getField("TAG"); + assertEquals(annoClazz.getName(), field.get(null)); + } + } +} From a534cb1af78ed27882dead652307a000a0d05629 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Sat, 20 Sep 2025 17:06:40 +0200 Subject: [PATCH 071/336] HBASE-29612 Remove HBaseTestingUtil.forceChangeTaskLogDir (#7329) Co-authored-by: Daniel Roudnitsky Signed-off-by: Duo Zhang (cherry picked from commit 99f8eabe19b4f9ef8e10ea0f0bea0a67d78436dc) --- .../hadoop/hbase/backup/TestBackupBase.java | 4 +- .../hbase/backup/TestBackupHFileCleaner.java | 4 +- .../hbase/backup/TestBackupSmallTests.java | 4 +- .../hadoop/hbase/HBaseTestingUtility.java | 39 +++---------------- 4 files changed, 12 insertions(+), 39 deletions(-) diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 2a0be003cab9..feb403ae5492 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -315,6 +315,8 @@ public static void setUpHelper() throws Exception { // Set MultiWAL (with 2 default WAL files per RS) conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); + conf1 = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniMapReduceCluster(); if (useSecondCluster) { conf2 = HBaseConfiguration.create(conf1); @@ -327,9 +329,7 @@ public static void setUpHelper() throws Exception { CommonFSUtils.setWALRootDir(TEST_UTIL2.getConfiguration(), p); TEST_UTIL2.startMiniCluster(); } - conf1 = TEST_UTIL.getConfiguration(); - TEST_UTIL.startMiniMapReduceCluster(); BACKUP_ROOT_DIR = new Path(new Path(TEST_UTIL.getConfiguration().get("fs.defaultFS")), BACKUP_ROOT_DIR) .toString(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java index 7fba9c02e94e..bfc729aa6792 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -46,7 +46,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({ MasterTests.class, SmallTests.class }) +@Category({ MasterTests.class, MediumTests.class }) public class TestBackupHFileCleaner { @ClassRule diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java index 83cc19578ade..5add9412014f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSmallTests.java @@ -22,14 +22,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.UserGroupInformation; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category(SmallTests.class) +@Category(MediumTests.class) public class TestBackupSmallTests extends TestBackupBase { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 3db73e1ad5f2..bc49fe406388 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -28,7 +28,6 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.net.BindException; import java.net.DatagramSocket; import java.net.InetAddress; @@ -139,7 +138,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.RegionSplitter; import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm; import org.apache.hadoop.hbase.util.RetryCounter; @@ -157,7 +155,6 @@ import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.yetus.audience.InterfaceAudience; @@ -2784,6 +2781,9 @@ public HRegionServer getRSForFirstRegionInTable(TableName tableName) /** * Starts a MiniMRCluster with a default number of TaskTracker's. + * MiniMRCluster caches hadoop.log.dir when first started. It is not possible to start multiple + * MiniMRCluster instances with different log dirs. MiniMRCluster is only to be used from when the + * test is run from a separate VM (i.e not in SmallTests) * @throws IOException When starting the cluster fails. */ public MiniMRCluster startMiniMapReduceCluster() throws IOException { @@ -2794,36 +2794,11 @@ public MiniMRCluster startMiniMapReduceCluster() throws IOException { return mrCluster; } - /** - * Tasktracker has a bug where changing the hadoop.log.dir system property will not change its - * internal static LOG_DIR variable. - */ - private void forceChangeTaskLogDir() { - Field logDirField; - try { - logDirField = TaskLog.class.getDeclaredField("LOG_DIR"); - logDirField.setAccessible(true); - - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - modifiersField.setInt(logDirField, logDirField.getModifiers() & ~Modifier.FINAL); - - logDirField.set(null, new File(hadoopLogDir, "userlogs")); - } catch (SecurityException e) { - throw new RuntimeException(e); - } catch (NoSuchFieldException e) { - // TODO Auto-generated catch block - throw new RuntimeException(e); - } catch (IllegalArgumentException e) { - throw new RuntimeException(e); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - /** * Starts a MiniMRCluster. Call {@link #setFileSystemURI(String)} to use a different - * filesystem. + * filesystem. MiniMRCluster caches hadoop.log.dir when first started. It is not possible to start + * multiple MiniMRCluster instances with different log dirs. MiniMRCluster is only to be used from + * when the test is run from a separate VM (i.e not in SmallTests) * @param servers The number of TaskTracker's to start. * @throws IOException When starting the cluster fails. */ @@ -2835,8 +2810,6 @@ private void startMiniMapReduceCluster(final int servers) throws IOException { setupClusterTestDir(); createDirsAndSetProperties(); - forceChangeTaskLogDir(); - //// hadoop2 specific settings // Tests were failing because this process used 6GB of virtual memory and was getting killed. // we up the VM usable so that processes don't get killed. From b1269b1473280d45c0b338c49f0447ca242e284d Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Mon, 22 Sep 2025 16:11:48 +0800 Subject: [PATCH 072/336] HBASE-29576 Replicate HBaseClassTestRule functionality for Junit 5 (#7331) Signed-off-by: Istvan Toth (cherry picked from commit 1cd9f29786127f4a6935f4e034d94ea083b12964) --- .../hadoop/hbase/HBaseJupiterExtension.java | 212 ++++++++++++++++++ .../hadoop/hbase/TestJUnit5TagConstants.java | 4 - .../org.junit.jupiter.api.extension.Extension | 16 ++ .../hadoop/hbase/http/TestLdapAdminACL.java | 3 - .../hadoop/hbase/http/TestLdapHttpServer.java | 3 - pom.xml | 1 + 6 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java create mode 100644 hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java new file mode 100644 index 000000000000..ff2ad14fe76b --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.hadoop.hbase.testclassification.IntegrationTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.yetus.audience.InterfaceAudience; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Store; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.platform.commons.JUnitException; +import org.junit.platform.commons.util.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Class test rule implementation for JUnit5. + *

+ * It ensures that all JUnit5 tests should have at least one of {@link SmallTests}, + * {@link MediumTests}, {@link LargeTests}, {@link IntegrationTests} tags, and set timeout based on + * the tag. + *

+ * It also controls the timeout for the whole test class running, while the timeout annotation in + * JUnit5 can only enforce the timeout for each test method. + *

+ * Finally, it also forbid System.exit call in tests. TODO: need to find a new way as + * SecurityManager has been removed since Java 21. + */ +@InterfaceAudience.Private +public class HBaseJupiterExtension + implements InvocationInterceptor, BeforeAllCallback, AfterAllCallback { + + private static final Logger LOG = LoggerFactory.getLogger(HBaseJupiterExtension.class); + + private static final SecurityManager securityManager = new TestSecurityManager(); + + private static final ExtensionContext.Namespace NAMESPACE = + ExtensionContext.Namespace.create(HBaseJupiterExtension.class); + + private static final Map TAG_TO_TIMEOUT = + ImmutableMap.of(SmallTests.TAG, Duration.ofMinutes(3), MediumTests.TAG, Duration.ofMinutes(6), + LargeTests.TAG, Duration.ofMinutes(13), IntegrationTests.TAG, Duration.ZERO); + + private static final String EXECUTOR = "executor"; + + private static final String DEADLINE = "deadline"; + + private Duration pickTimeout(ExtensionContext ctx) { + Set timeoutTags = TAG_TO_TIMEOUT.keySet(); + Set timeoutTag = Sets.intersection(timeoutTags, ctx.getTags()); + if (timeoutTag.isEmpty()) { + fail("Test class " + ctx.getDisplayName() + " does not have any of the following scale tags " + + timeoutTags); + } + if (timeoutTag.size() > 1) { + fail("Test class " + ctx.getDisplayName() + " has multiple scale tags " + timeoutTag); + } + return TAG_TO_TIMEOUT.get(Iterables.getOnlyElement(timeoutTag)); + } + + @Override + public void beforeAll(ExtensionContext ctx) throws Exception { + // TODO: remove this usage + System.setSecurityManager(securityManager); + Duration timeout = pickTimeout(ctx); + if (timeout.isZero() || timeout.isNegative()) { + LOG.info("No timeout for {}", ctx.getDisplayName()); + // zero means no timeout + return; + } + Instant deadline = Instant.now().plus(timeout); + LOG.info("Timeout for {} is {}, it should be finished before {}", ctx.getDisplayName(), timeout, + deadline); + ExecutorService executor = + Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("HBase-Test-" + ctx.getDisplayName() + "-Main-Thread").build()); + Store store = ctx.getStore(NAMESPACE); + store.put(EXECUTOR, executor); + store.put(DEADLINE, deadline); + } + + @Override + public void afterAll(ExtensionContext ctx) throws Exception { + Store store = ctx.getStore(NAMESPACE); + ExecutorService executor = store.remove(EXECUTOR, ExecutorService.class); + if (executor != null) { + executor.shutdownNow(); + } + store.remove(DEADLINE); + // reset secutiry manager + System.setSecurityManager(null); + } + + private T runWithTimeout(Invocation invocation, ExtensionContext ctx) throws Throwable { + Store store = ctx.getStore(NAMESPACE); + ExecutorService executor = store.get(EXECUTOR, ExecutorService.class); + if (executor == null) { + return invocation.proceed(); + } + Instant deadline = store.get(DEADLINE, Instant.class); + Instant now = Instant.now(); + if (!now.isBefore(deadline)) { + fail("Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline); + return null; + } + + Duration remaining = Duration.between(now, deadline); + LOG.info("remaining timeout for {} is {}", ctx.getDisplayName(), remaining); + Future future = executor.submit(() -> { + try { + return invocation.proceed(); + } catch (Throwable t) { + // follow the same pattern with junit5 + throw ExceptionUtils.throwAsUncheckedException(t); + } + }); + try { + return future.get(remaining.toNanos(), TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Test " + ctx.getDisplayName() + " interrupted"); + return null; + } catch (ExecutionException e) { + throw ExceptionUtils.throwAsUncheckedException(e.getCause()); + } catch (TimeoutException e) { + + throw new JUnitException( + "Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline, e); + } + } + + @Override + public void interceptBeforeAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext); + } + + @Override + public void interceptBeforeEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext); + } + + @Override + public void interceptTestMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext); + } + + @Override + public void interceptAfterEachMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext); + } + + @Override + public void interceptAfterAllMethod(Invocation invocation, + ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) + throws Throwable { + runWithTimeout(invocation, extensionContext); + } + + @Override + public T interceptTestClassConstructor(Invocation invocation, + ReflectiveInvocationContext> invocationContext, + ExtensionContext extensionContext) throws Throwable { + return runWithTimeout(invocation, extensionContext); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java index 43607e171817..3e30b388ab2e 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestJUnit5TagConstants.java @@ -20,21 +20,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.lang.reflect.Field; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; /** * Verify that the values are all correct. */ @Tag(MiscTests.TAG) @Tag(SmallTests.TAG) -// TODO: this is the timeout for each method, not the whole class -@Timeout(value = 1, unit = TimeUnit.MINUTES) public class TestJUnit5TagConstants { @Test diff --git a/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 000000000000..0cb8a35a1ee8 --- /dev/null +++ b/hbase-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.hadoop.hbase.HBaseJupiterExtension diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java index c4fd208fa7ce..91a3321bdfce 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapAdminACL.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.net.HttpURLConnection; -import java.util.concurrent.TimeUnit; import org.apache.directory.server.annotations.CreateLdapServer; import org.apache.directory.server.annotations.CreateTransport; import org.apache.directory.server.core.annotations.ApplyLdifs; @@ -36,7 +35,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +54,6 @@ "dn: uid=jdoe," + LdapConstants.LDAP_BASE_DN, "cn: John Doe", "sn: Doe", "objectClass: inetOrgPerson", "uid: jdoe", "userPassword: secure123" }) -@Timeout(value = 1, unit = TimeUnit.MINUTES) public class TestLdapAdminACL extends LdapServerTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestLdapAdminACL.class); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java index 9faa8dc49fb6..c4936513fb36 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestLdapHttpServer.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.net.HttpURLConnection; -import java.util.concurrent.TimeUnit; import org.apache.directory.server.annotations.CreateLdapServer; import org.apache.directory.server.annotations.CreateTransport; import org.apache.directory.server.core.annotations.ApplyLdifs; @@ -32,7 +31,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; /** * Test class for LDAP authentication on the HttpServer. @@ -47,7 +45,6 @@ + "dc: example\n" + "objectClass: top\n" + "objectClass: domain\n\n")) }) @ApplyLdifs({ "dn: uid=bjones," + LdapConstants.LDAP_BASE_DN, "cn: Bob Jones", "sn: Jones", "objectClass: inetOrgPerson", "uid: bjones", "userPassword: p@ssw0rd" }) -@Timeout(value = 1, unit = TimeUnit.MINUTES) public class TestLdapHttpServer extends LdapServerTestBase { private static final String BJONES_CREDENTIALS = "bjones:p@ssw0rd"; diff --git a/pom.xml b/pom.xml index fe8249af5e5c..5b57dcdcd8ec 100644 --- a/pom.xml +++ b/pom.xml @@ -1713,6 +1713,7 @@ listener org.apache.hadoop.hbase.TimedOutTestsListener,org.apache.hadoop.hbase.HBaseClassTestRuleChecker,org.apache.hadoop.hbase.ResourceCheckerJUnitListener + junit.jupiter.extensions.autodetection.enabled=true From df1bfc0d94b2b87b9d763ad608b255ac47010956 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Mon, 22 Sep 2025 16:18:06 +0200 Subject: [PATCH 073/336] HBASE-29550 Reflection error in TestRSGroupsKillRS with Java 21 (#7328) Signed-off-by: Duo Zhang (cherry picked from commit cac063aaa15170e201ecb99314f3f5d0ed9f1387) --- .../apache/hadoop/hbase/trace/TraceUtil.java | 4 +- .../apache/hadoop/hbase/util/VersionInfo.java | 5 ++- .../hbase/rsgroup/TestRSGroupsKillRS.java | 39 ++++++++++--------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java index 5b1fb86a351a..260c0064f840 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/TraceUtil.java @@ -28,8 +28,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; -import org.apache.hadoop.hbase.Version; import org.apache.hadoop.hbase.util.FutureUtils; +import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private @@ -39,7 +39,7 @@ private TraceUtil() { } public static Tracer getGlobalTracer() { - return GlobalOpenTelemetry.getTracer("org.apache.hbase", Version.version); + return GlobalOpenTelemetry.getTracer("org.apache.hbase", VersionInfo.getVersion()); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java index 60a37af60483..80ae42a3a3b9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java @@ -40,12 +40,15 @@ public class VersionInfo { // higher than any numbers in the version. private static final int VERY_LARGE_NUMBER = 100000; + // Copying into a non-final member so that it can be changed by reflection for testing + private static String version = Version.version; + /** * Get the hbase version. * @return the hbase version string, eg. "0.6.3-dev" */ public static String getVersion() { - return Version.version; + return version; } /** diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java index e11d8f5717cf..6caf0a744e38 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue; import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -46,7 +45,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.VersionInfo; import org.junit.After; import org.junit.AfterClass; @@ -266,24 +264,27 @@ public void testLowerMetaGroupVersion() throws Exception { Address address = servers.iterator().next(); int majorVersion = VersionInfo.getMajorVersion(originVersion); assertTrue(majorVersion >= 1); - String lowerVersion = String.valueOf(majorVersion - 1) + originVersion.split("\\.")[1]; - setFinalStatic(Version.class.getField("version"), lowerVersion); - TEST_UTIL.getMiniHBaseCluster().startRegionServer(address.getHostName(), address.getPort()); - assertEquals(NUM_SLAVES_BASE, - TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); - assertTrue(VersionInfo.compareVersion(originVersion, - master.getRegionServerVersion(getServerName(servers.iterator().next()))) > 0); - LOG.debug("wait for META assigned..."); - // SCP finished, which means all regions assigned too. - TEST_UTIL.waitFor(60000, () -> !TEST_UTIL.getHBaseCluster().getMaster().getProcedures().stream() - .filter(p -> (p instanceof ServerCrashProcedure)).findAny().isPresent()); + String lowerVersion = + String.valueOf(majorVersion - 1) + originVersion.substring(originVersion.indexOf(".")); + try { + setVersionInfoVersion(lowerVersion); + TEST_UTIL.getMiniHBaseCluster().startRegionServer(address.getHostName(), address.getPort()); + assertEquals(NUM_SLAVES_BASE, + TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size()); + assertTrue(VersionInfo.compareVersion(originVersion, + master.getRegionServerVersion(getServerName(servers.iterator().next()))) > 0); + LOG.debug("wait for META assigned..."); + // SCP finished, which means all regions assigned too. + TEST_UTIL.waitFor(60000, () -> !TEST_UTIL.getHBaseCluster().getMaster().getProcedures() + .stream().filter(p -> (p instanceof ServerCrashProcedure)).findAny().isPresent()); + } finally { + setVersionInfoVersion(Version.version); + } } - private static void setFinalStatic(Field field, Object newValue) throws Exception { - field.setAccessible(true); - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(null, newValue); + private static void setVersionInfoVersion(String newValue) throws Exception { + Field f = VersionInfo.class.getDeclaredField("version"); + f.setAccessible(true); + f.set(null, newValue); } } From 84c056b43fda1a0f7baf2a820274f05dae0e0d0b Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Fri, 26 Sep 2025 10:28:14 +0100 Subject: [PATCH 074/336] HBASE-29627 Handle any block cache fetching errors when reading a block in HFileReaderImpl (#7341) (#7344) Signed-off-by: Peter Somogyi --- .../hbase/io/hfile/HFileReaderImpl.java | 26 +++++++++++++++++++ .../hbase/io/hfile/TestHFileReaderImpl.java | 22 ++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index e837e8f1bd6b..df6821f1fd3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -1180,6 +1180,32 @@ public HFileBlock getCachedBlock(BlockCacheKey cacheKey, boolean cacheBlock, boo } return cachedBlock; } + } catch (Exception e) { + if (cachedBlock != null) { + returnAndEvictBlock(cache, cacheKey, cachedBlock); + } + LOG.warn("Failed retrieving block from cache with key {}. " + + "\n Evicting this block from cache and will read it from file system. " + + "\n Exception details: ", cacheKey, e); + if (LOG.isDebugEnabled()) { + LOG.debug("Further tracing details for failed block cache retrieval:" + + "\n Complete File path - {}," + "\n Expected Block Type - {}, Actual Block Type - {}," + + "\n Cache compressed - {}" + "\n Header size (after deserialized from cache) - {}" + + "\n Size with header - {}" + "\n Uncompressed size without header - {} " + + "\n Total byte buffer size - {}" + "\n Encoding code - {}", this.path, + expectedBlockType, (cachedBlock != null ? cachedBlock.getBlockType() : "N/A"), + (expectedBlockType != null + ? cacheConf.shouldCacheCompressed(expectedBlockType.getCategory()) + : "N/A"), + (cachedBlock != null ? cachedBlock.headerSize() : "N/A"), + (cachedBlock != null ? cachedBlock.getOnDiskSizeWithHeader() : "N/A"), + (cachedBlock != null ? cachedBlock.getUncompressedSizeWithoutHeader() : "N/A"), + (cachedBlock != null ? cachedBlock.getBufferReadOnly().limit() : "N/A"), + (cachedBlock != null + ? cachedBlock.getBufferReadOnly().getShort(cachedBlock.headerSize()) + : "N/A")); + } + return null; } finally { // Count bytes read as cached block is being returned if (isScanMetricsEnabled && cachedBlock != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java index 42f2cf5ebd9d..1eb5ac02607f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java @@ -18,7 +18,12 @@ package org.apache.hadoop.hbase.io.hfile; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; @@ -116,6 +121,23 @@ public void testRecordBlockSize() throws IOException { } } + @Test + public void testReadWorksWhenCacheCorrupt() throws Exception { + BlockCache mockedCache = mock(BlockCache.class); + when(mockedCache.getBlock(any(), anyBoolean(), anyBoolean(), anyBoolean())) + .thenThrow(new RuntimeException("Injected error")); + Path p = makeNewFile(); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); + HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf, mockedCache), true, conf); + long offset = 0; + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null, false); + assertNotNull(block); + offset += block.getOnDiskSizeWithHeader(); + } + } + @Test public void testSeekBefore() throws Exception { Path p = makeNewFile(); From e7154e1c4c664ecaa56f349cee0af8fb25945f7b Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Thu, 25 Sep 2025 10:44:48 +0100 Subject: [PATCH 075/336] HBASE-29623 Blocks for CFs with BlockCache disabled may still get cached on write or compaction (#7339) Signed-off-by: Peter Somogyi Change-Id: If4e4efd89acec4fb6941de76fafa89240d6207d5 --- .../hadoop/hbase/io/hfile/CacheConfig.java | 61 ++++++++++--------- .../hbase/io/hfile/TestCacheConfig.java | 21 +++++-- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index b6357ca94b86..ae196340db61 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -125,7 +125,7 @@ public class CacheConfig implements PropagatingConfigurationObserver { private volatile boolean cacheDataOnRead; /** Whether blocks should be flagged as in-memory when being cached */ - private final boolean inMemory; + private boolean inMemory; /** Whether data blocks should be cached when new files are written */ private volatile boolean cacheDataOnWrite; @@ -140,28 +140,29 @@ public class CacheConfig implements PropagatingConfigurationObserver { private volatile boolean evictOnClose; /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */ - private final boolean cacheDataCompressed; + private boolean cacheDataCompressed; /** Whether data blocks should be prefetched into the cache */ - private final boolean prefetchOnOpen; + private boolean prefetchOnOpen; /** * Whether data blocks should be cached when compacted file is written */ - private final boolean cacheCompactedDataOnWrite; + private boolean cacheCompactedDataOnWrite; /** * Determine threshold beyond which we do not cache blocks on compaction */ private long cacheCompactedDataOnWriteThreshold; - private final boolean dropBehindCompaction; + private boolean dropBehindCompaction; // Local reference to the block cache private final BlockCache blockCache; private final ByteBuffAllocator byteBuffAllocator; + /** * Create a cache configuration using the specified configuration object and defaults for family * level settings. Only use if no column family context. @@ -182,30 +183,32 @@ public CacheConfig(Configuration conf, BlockCache blockCache) { */ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family, BlockCache blockCache, ByteBuffAllocator byteBuffAllocator) { - this.cacheDataOnRead = conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ) - && (family == null ? true : family.isBlockCacheEnabled()); - this.inMemory = family == null ? DEFAULT_IN_MEMORY : family.isInMemory(); - this.cacheDataCompressed = - conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED); - this.dropBehindCompaction = - conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT); - // For the following flags we enable them regardless of per-schema settings - // if they are enabled in the global configuration. - this.cacheDataOnWrite = conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) - || (family == null ? false : family.isCacheDataOnWrite()); - this.cacheIndexesOnWrite = - conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) - || (family == null ? false : family.isCacheIndexesOnWrite()); - this.cacheBloomsOnWrite = - conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) - || (family == null ? false : family.isCacheBloomsOnWrite()); - this.evictOnClose = conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) - || (family == null ? false : family.isEvictBlocksOnClose()); - this.prefetchOnOpen = conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) - || (family == null ? false : family.isPrefetchBlocksOnOpen()); - this.cacheCompactedDataOnWrite = - conf.getBoolean(CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE); - this.cacheCompactedDataOnWriteThreshold = getCacheCompactedBlocksOnWriteThreshold(conf); + if (family == null || family.isBlockCacheEnabled()) { + this.cacheDataOnRead = conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ); + this.inMemory = family == null ? DEFAULT_IN_MEMORY : family.isInMemory(); + this.cacheDataCompressed = + conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED); + this.dropBehindCompaction = + conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT); + // For the following flags we enable them regardless of per-schema settings + // if they are enabled in the global configuration. + this.cacheDataOnWrite = + conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) + || (family == null ? false : family.isCacheDataOnWrite()); + this.cacheIndexesOnWrite = + conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) + || (family == null ? false : family.isCacheIndexesOnWrite()); + this.cacheBloomsOnWrite = + conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) + || (family == null ? false : family.isCacheBloomsOnWrite()); + this.evictOnClose = conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) + || (family == null ? false : family.isEvictBlocksOnClose()); + this.prefetchOnOpen = conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) + || (family == null ? false : family.isPrefetchBlocksOnOpen()); + this.cacheCompactedDataOnWrite = conf.getBoolean(CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, + DEFAULT_CACHE_COMPACTED_BLOCKS_ON_WRITE); + this.cacheCompactedDataOnWriteThreshold = getCacheCompactedBlocksOnWriteThreshold(conf); + } this.blockCache = blockCache; this.byteBuffAllocator = byteBuffAllocator; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index bee8ca0667de..c26c8008a31e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -188,12 +188,14 @@ void basicBlockCacheOps(final BlockCache bc, final CacheConfig cc, final boolean @Test public void testDisableCacheDataBlock() throws IOException { + // First tests the default configs behaviour and block cache enabled Configuration conf = HBaseConfiguration.create(); CacheConfig cacheConfig = new CacheConfig(conf); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); assertFalse(cacheConfig.shouldCacheDataCompressed()); assertFalse(cacheConfig.shouldCacheDataOnWrite()); + assertFalse(cacheConfig.shouldCacheCompactedBlocksOnWrite()); assertTrue(cacheConfig.shouldCacheDataOnRead()); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); @@ -201,10 +203,12 @@ public void testDisableCacheDataBlock() throws IOException { assertFalse(cacheConfig.shouldCacheBloomsOnWrite()); assertFalse(cacheConfig.shouldCacheIndexesOnWrite()); + // Tests block cache enabled and related cache on write flags enabled conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, true); cacheConfig = new CacheConfig(conf); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); @@ -217,9 +221,12 @@ public void testDisableCacheDataBlock() throws IOException { assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); + assertTrue(cacheConfig.shouldCacheCompactedBlocksOnWrite()); + // Tests block cache enabled but related cache on read/write properties disabled conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, false); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + conf.setBoolean(CacheConfig.CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, false); cacheConfig = new CacheConfig(conf); assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); @@ -227,14 +234,20 @@ public void testDisableCacheDataBlock() throws IOException { assertFalse(cacheConfig.shouldCacheDataCompressed()); assertFalse(cacheConfig.shouldCacheDataOnWrite()); assertFalse(cacheConfig.shouldCacheDataOnRead()); + assertFalse(cacheConfig.shouldCacheCompactedBlocksOnWrite()); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); - conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, true); - conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + // Finally tests block cache disabled in the column family but all cache on read/write + // properties enabled in the config. + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true); + conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_COMPACTED_BLOCKS_ON_WRITE_KEY, true); HColumnDescriptor family = new HColumnDescriptor("testDisableCacheDataBlock"); family.setBlockCacheEnabled(false); @@ -248,8 +261,8 @@ public void testDisableCacheDataBlock() throws IOException { assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); - assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); - assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); + assertFalse(cacheConfig.shouldCacheBloomsOnWrite()); + assertFalse(cacheConfig.shouldCacheIndexesOnWrite()); } @Test From 5312aa163c523da359edece2243f0a54a2b8a962 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Mon, 29 Sep 2025 15:08:35 -0400 Subject: [PATCH 076/336] HBASE-28440 Add support for using mapreduce sort in HFileOutputFormat2 (#7295) (#7342) Signed-off-by: Ray Mattingly Co-authored-by: Hernan Romer Co-authored-by: Hernan Gelaf-Romer --- .../impl/IncrementalTableBackupClient.java | 12 ++ .../mapreduce/MapReduceHFileSplitterJob.java | 36 +++- .../hbase/mapreduce/HFileOutputFormat2.java | 32 +++- .../apache/hadoop/hbase/mapreduce/Import.java | 4 + .../mapreduce/KeyOnlyCellComparable.java | 94 ++++++++++ .../mapreduce/PreSortedCellsReducer.java | 46 +++++ .../hadoop/hbase/mapreduce/WALPlayer.java | 37 +++- .../mapreduce/TestCellBasedWALPlayer2.java | 3 +- .../hadoop/hbase/mapreduce/TestWALPlayer.java | 167 +++++++++++++----- 9 files changed, 373 insertions(+), 58 deletions(-) create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index d51f1f471514..ae32a8dbeb51 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -335,6 +335,7 @@ public void execute() throws IOException, ColumnFamilyMismatchException { } protected void incrementalCopyHFiles(String[] files, String backupDest) throws IOException { + boolean diskBasedSortingOriginalValue = HFileOutputFormat2.diskBasedSortingEnabled(conf); try { LOG.debug("Incremental copy HFiles is starting. dest=" + backupDest); // set overall backup phase: incremental_copy @@ -349,6 +350,7 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I LOG.debug("Setting incremental copy HFiles job name to : " + jobname); } conf.set(JOB_NAME_CONF_KEY, jobname); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); int res = copyService.copy(backupInfo, backupManager, conf, BackupType.INCREMENTAL, strArr); @@ -361,6 +363,8 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I + " finished."); } finally { deleteBulkLoadDirectory(); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + diskBasedSortingOriginalValue); } } @@ -415,6 +419,9 @@ protected void walToHFiles(List dirPaths, List tableList) throws conf.setBoolean(HFileOutputFormat2.TABLE_NAME_WITH_NAMESPACE_INCLUSIVE_KEY, true); conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); conf.set(JOB_NAME_CONF_KEY, jobname); + + boolean diskBasedSortingEnabledOriginalValue = HFileOutputFormat2.diskBasedSortingEnabled(conf); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); String[] playerArgs = { dirs, StringUtils.join(tableList, ",") }; try { @@ -430,6 +437,11 @@ protected void walToHFiles(List dirPaths, List tableList) throws } catch (Exception ee) { throw new IOException("Can not convert from directory " + dirs + " (check Hadoop, HBase and WALPlayer M/R job logs) ", ee); + } finally { + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + diskBasedSortingEnabledOriginalValue); + conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY); + conf.unset(JOB_NAME_CONF_KEY); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java index 28db0c605f79..4f3a7f925c63 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -33,11 +34,14 @@ import org.apache.hadoop.hbase.mapreduce.CellSortReducer; import org.apache.hadoop.hbase.mapreduce.HFileInputFormat; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.hbase.mapreduce.KeyOnlyCellComparable; +import org.apache.hadoop.hbase.mapreduce.PreSortedCellsReducer; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MapReduceExtendedCell; import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -71,18 +75,28 @@ protected MapReduceHFileSplitterJob(final Configuration c) { /** * A mapper that just writes out cells. This one can be used together with {@link CellSortReducer} */ - static class HFileCellMapper extends Mapper { + static class HFileCellMapper extends Mapper, Cell> { + + private boolean diskBasedSortingEnabled = false; @Override public void map(NullWritable key, Cell value, Context context) throws IOException, InterruptedException { - context.write(new ImmutableBytesWritable(CellUtil.cloneRow(value)), - new MapReduceExtendedCell(value)); + ExtendedCell extendedCell = (ExtendedCell) value; + context.write(wrap(extendedCell), new MapReduceExtendedCell(extendedCell)); } @Override public void setup(Context context) throws IOException { - // do nothing + diskBasedSortingEnabled = + HFileOutputFormat2.diskBasedSortingEnabled(context.getConfiguration()); + } + + private WritableComparable wrap(ExtendedCell cell) { + if (diskBasedSortingEnabled) { + return new KeyOnlyCellComparable(cell); + } + return new ImmutableBytesWritable(CellUtil.cloneRow(cell)); } } @@ -106,13 +120,23 @@ public Job createSubmittableJob(String[] args) throws IOException { true); job.setJarByClass(MapReduceHFileSplitterJob.class); job.setInputFormatClass(HFileInputFormat.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); TableName tableName = TableName.valueOf(tabName); job.setMapperClass(HFileCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + } else { + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index cb2c62601712..2906238edc79 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -83,6 +84,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; @@ -194,6 +196,11 @@ protected static byte[] combineTableNameSuffix(byte[] tableName, byte[] suffix) "hbase.mapreduce.hfileoutputformat.extendedcell.enabled"; static final boolean EXTENDED_CELL_SERIALIZATION_ENABLED_DEFULT = false; + @InterfaceAudience.Private + public static final String DISK_BASED_SORTING_ENABLED_KEY = + "hbase.mapreduce.hfileoutputformat.disk.based.sorting.enabled"; + private static final boolean DISK_BASED_SORTING_ENABLED_DEFAULT = false; + public static final String REMOTE_CLUSTER_CONF_PREFIX = "hbase.hfileoutputformat.remote.cluster."; public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY = REMOTE_CLUSTER_CONF_PREFIX + "zookeeper.quorum"; @@ -579,12 +586,19 @@ private static void writePartitions(Configuration conf, Path partitionsPath, // Write the actual file FileSystem fs = partitionsPath.getFileSystem(conf); - SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, partitionsPath, - ImmutableBytesWritable.class, NullWritable.class); + boolean diskBasedSortingEnabled = diskBasedSortingEnabled(conf); + Class keyClass = + diskBasedSortingEnabled ? KeyOnlyCellComparable.class : ImmutableBytesWritable.class; + SequenceFile.Writer writer = + SequenceFile.createWriter(fs, conf, partitionsPath, keyClass, NullWritable.class); try { for (ImmutableBytesWritable startKey : sorted) { - writer.append(startKey, NullWritable.get()); + Writable writable = diskBasedSortingEnabled + ? new KeyOnlyCellComparable(KeyValueUtil.createFirstOnRow(startKey.get())) + : startKey; + + writer.append(writable, NullWritable.get()); } } finally { writer.close(); @@ -631,6 +645,10 @@ public static void configureIncrementalLoad(Job job, TableDescriptor tableDescri configureIncrementalLoad(job, singleTableInfo, HFileOutputFormat2.class); } + public static boolean diskBasedSortingEnabled(Configuration conf) { + return conf.getBoolean(DISK_BASED_SORTING_ENABLED_KEY, DISK_BASED_SORTING_ENABLED_DEFAULT); + } + static void configureIncrementalLoad(Job job, List multiTableInfo, Class> cls) throws IOException { Configuration conf = job.getConfiguration(); @@ -652,7 +670,13 @@ static void configureIncrementalLoad(Job job, List multiTableInfo, // Based on the configured map output class, set the correct reducer to properly // sort the incoming values. // TODO it would be nice to pick one or the other of these formats. - if ( + boolean diskBasedSorting = diskBasedSortingEnabled(conf); + + if (diskBasedSorting) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + job.setReducerClass(PreSortedCellsReducer.class); + } else if ( KeyValue.class.equals(job.getMapOutputValueClass()) || MapReduceExtendedCell.class.equals(job.getMapOutputValueClass()) ) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index 4adcfbfcd3f6..03abcf159753 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -200,6 +200,10 @@ public CellWritableComparable(Cell kv) { this.kv = kv; } + public Cell getCell() { + return kv; + } + @Override public void write(DataOutput out) throws IOException { int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(kv); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java new file mode 100644 index 000000000000..d9b28f8a6895 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableComparator; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class KeyOnlyCellComparable implements WritableComparable { + + static { + WritableComparator.define(KeyOnlyCellComparable.class, new KeyOnlyCellComparator()); + } + + private ExtendedCell cell = null; + + public KeyOnlyCellComparable() { + } + + public KeyOnlyCellComparable(ExtendedCell cell) { + this.cell = cell; + } + + public ExtendedCell getCell() { + return cell; + } + + @Override + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "EQ_COMPARETO_USE_OBJECT_EQUALS", + justification = "This is wrong, yes, but we should be purging Writables, not fixing them") + public int compareTo(KeyOnlyCellComparable o) { + return CellComparator.getInstance().compare(cell, o.cell); + } + + @Override + public void write(DataOutput out) throws IOException { + int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(cell); + int valueLen = 0; // We avoid writing value here. So just serialize as if an empty value. + out.writeInt(keyLen + valueLen + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + out.writeInt(keyLen); + out.writeInt(valueLen); + PrivateCellUtil.writeFlatKey(cell, out); + out.writeLong(cell.getSequenceId()); + } + + @Override + public void readFields(DataInput in) throws IOException { + cell = KeyValue.create(in); + long seqId = in.readLong(); + cell.setSequenceId(seqId); + } + + public static class KeyOnlyCellComparator extends WritableComparator { + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + try (DataInputStream d1 = new DataInputStream(new ByteArrayInputStream(b1, s1, l1)); + DataInputStream d2 = new DataInputStream(new ByteArrayInputStream(b2, s2, l2))) { + KeyOnlyCellComparable kv1 = new KeyOnlyCellComparable(); + kv1.readFields(d1); + KeyOnlyCellComparable kv2 = new KeyOnlyCellComparable(); + kv2.readFields(d2); + return compare(kv1, kv2); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java new file mode 100644 index 000000000000..81871ffb59c2 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.MapReduceExtendedCell; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class PreSortedCellsReducer + extends Reducer { + + @Override + protected void reduce(KeyOnlyCellComparable key, Iterable values, Context context) + throws IOException, InterruptedException { + + int index = 0; + for (Cell cell : values) { + context.write(new ImmutableBytesWritable(CellUtil.cloneRow(key.getCell())), + new MapReduceExtendedCell(cell)); + + if (++index % 100 == 0) { + context.setStatus("Wrote " + index + " cells"); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index e06300848f68..cc5820c99b58 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; @@ -53,6 +54,7 @@ import org.apache.hadoop.hbase.util.MapReduceExtendedCell; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -138,9 +140,10 @@ public void setup(Context context) throws IOException { /** * A mapper that just writes out Cells. This one can be used together with {@link CellSortReducer} */ - static class WALCellMapper extends Mapper { + static class WALCellMapper extends Mapper, Cell> { private Set tableSet = new HashSet<>(); private boolean multiTableSupport = false; + private boolean diskBasedSortingEnabled = false; @Override public void map(WALKey key, WALEdit value, Context context) throws IOException { @@ -161,7 +164,8 @@ public void map(WALKey key, WALEdit value, Context context) throws IOException { byte[] outKey = multiTableSupport ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell)) : CellUtil.cloneRow(cell); - context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell)); + ExtendedCell extendedCell = (ExtendedCell) cell; + context.write(wrapKey(outKey, extendedCell), new MapReduceExtendedCell(extendedCell)); } } } catch (InterruptedException e) { @@ -174,8 +178,23 @@ public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); String[] tables = conf.getStrings(TABLES_KEY); this.multiTableSupport = conf.getBoolean(MULTI_TABLES_SUPPORT, false); + this.diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); Collections.addAll(tableSet, tables); } + + private WritableComparable wrapKey(byte[] key, ExtendedCell cell) { + if (this.diskBasedSortingEnabled) { + // Important to build a new cell with the updated key to maintain multi-table support + KeyValue kv = new KeyValue(key, 0, key.length, cell.getFamilyArray(), + cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), + cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(), + KeyValue.Type.codeToType(cell.getTypeByte()), null, 0, 0); + kv.setSequenceId(cell.getSequenceId()); + return new KeyOnlyCellComparable(kv); + } else { + return new ImmutableBytesWritable(key); + } + } } /** @@ -353,7 +372,13 @@ public Job createSubmittableJob(String[] args) throws IOException { job.setJarByClass(WALPlayer.class); job.setInputFormatClass(WALInputFormat.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); if (hfileOutPath != null) { @@ -372,7 +397,11 @@ public Job createSubmittableJob(String[] args) throws IOException { List tableNames = getTableNameList(tables); job.setMapperClass(WALCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + } else { + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java index 283acbabf6e4..d6c4b623ad42 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.util.ToolRunner; @@ -172,7 +173,7 @@ private void testWALKeyValueMapper(final String tableConfigKey) throws Exception WALKey key = mock(WALKey.class); when(key.getTableName()).thenReturn(TableName.valueOf("table")); @SuppressWarnings("unchecked") - Mapper.Context context = mock(Context.class); + Mapper, Cell>.Context context = mock(Context.class); when(context.getConfiguration()).thenReturn(configuration); WALEdit value = mock(WALEdit.class); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index c6e51eee40ff..d2d9cc831dae 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -114,6 +114,50 @@ public static void afterClass() throws Exception { logFs.delete(walRootDir, true); } + @Test + public void testDiskBasedSortingEnabled() throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + final byte[] COLUMN2 = Bytes.toBytes("c2"); + final byte[] ROW = Bytes.toBytes("row"); + Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + p.addColumn(FAMILY, COLUMN2, COLUMN2); + t1.put(p); + // delete one column + Delete d = new Delete(ROW); + d.addColumns(FAMILY, COLUMN1); + t1.delete(d); + + // replay the WAL, map table 1 to table 2 + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + + Configuration configuration = TEST_UTIL.getConfiguration(); + configuration.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + WALPlayer player = new WALPlayer(configuration); + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + } + /** * Test that WALPlayer can replay recovered.edits files. */ @@ -123,19 +167,22 @@ public void testPlayingRecoveredEdit() throws Exception { TEST_UTIL.createTable(tn, TestRecoveredEdits.RECOVEREDEDITS_COLUMNFAMILY); // Copy testing recovered.edits file that is over under hbase-server test resources // up into a dir in our little hdfs cluster here. - String hbaseServerTestResourcesEdits = - System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" - + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); - assertTrue(new File(hbaseServerTestResourcesEdits).exists()); - FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); - // Target dir. - Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); - assertTrue(dfs.mkdirs(targetDir)); - dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); - assertEquals(0, - ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); - // I don't know how many edits are in this file for this table... so just check more than 1. - assertTrue(TEST_UTIL.countRows(tn) > 0); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String hbaseServerTestResourcesEdits = + System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" + + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); + assertTrue(new File(hbaseServerTestResourcesEdits).exists()); + FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); + // Target dir. + Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); + assertTrue(dfs.mkdirs(targetDir)); + dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); + assertEquals(0, + ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); + // I don't know how many edits are in this file for this table... so just check more than 1. + assertTrue(TEST_UTIL.countRows(tn) > 0); + dfs.delete(targetDir, true); + }); } /** @@ -150,7 +197,7 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { final byte[] column1 = Bytes.toBytes("c1"); final byte[] column2 = Bytes.toBytes("c2"); final byte[] row = Bytes.toBytes("row"); - Table table = TEST_UTIL.createTable(tableName, family); + final Table table = TEST_UTIL.createTable(tableName, family); long now = EnvironmentEdgeManager.currentTime(); // put a row into the first table @@ -187,29 +234,38 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { configuration.set(WALPlayer.BULK_OUTPUT_CONF_KEY, outPath); configuration.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); - WALPlayer player = new WALPlayer(configuration); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName.getNameAsString() })); + final byte[] finalLastVal = lastVal; + + runWithDiskBasedSortingDisabledAndEnabled(() -> { + WALPlayer player = new WALPlayer(configuration); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName.getNameAsString() })); - Get g = new Get(row); - Result result = table.get(g); - byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + Get g = new Get(row); + Result result = table.get(g); + byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); - table = TEST_UTIL.truncateTable(tableName); - g = new Get(row); - result = table.get(g); - assertThat(result.listCells(), nullValue()); + TEST_UTIL.truncateTable(tableName); + g = new Get(row); + result = table.get(g); + assertThat(result.listCells(), nullValue()); - BulkLoadHFiles.create(configuration).bulkLoad(tableName, - new Path(outPath, tableName.getNameAsString())); + BulkLoadHFiles.create(configuration).bulkLoad(tableName, + new Path(outPath, tableName.getNameAsString())); - g = new Get(row); - result = table.get(g); - value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + g = new Get(row); + result = table.get(g); + value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); - assertThat(result.listCells(), notNullValue()); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + assertThat(result.listCells(), notNullValue()); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); + + // cleanup + Path out = new Path(outPath); + FileSystem fs = out.getFileSystem(configuration); + assertTrue(fs.delete(out, true)); + }); } /** @@ -244,18 +300,21 @@ public void testWALPlayer() throws Exception { Configuration configuration = TEST_UTIL.getConfiguration(); WALPlayer player = new WALPlayer(configuration); - String optionName = "_test_.name"; - configuration.set(optionName, "1000"); - player.setupTime(configuration, optionName); - assertEquals(1000, configuration.getLong(optionName, 0)); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); - // verify the WAL was player into table 2 - Get g = new Get(ROW); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + }); } /** @@ -335,7 +394,29 @@ public void testMainMethod() throws Exception { System.setErr(oldPrintStream); System.setSecurityManager(SECURITY_MANAGER); } + } + + private static void runWithDiskBasedSortingDisabledAndEnabled(TestMethod method) + throws Exception { + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + false); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + true); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + } + private interface TestMethod { + void run() throws Exception; } } From 7342b931529f4722f0c485d3ecd2eb0a706a7e84 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Tue, 30 Sep 2025 04:13:38 -0700 Subject: [PATCH 077/336] HBASE-29629 Record the quota user name value on metrics for RpcThrottlingExceptions (#7345) Signed-off-by: Wellington Chevreuil --- .../main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java | 2 +- .../hadoop/hbase/quotas/RegionServerRpcQuotaManager.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 16681eb45f8f..325f31586c5e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -229,7 +229,7 @@ protected boolean isExceedThrottleQuotaEnabled() { * username * @param ugi The request's UserGroupInformation */ - private String getQuotaUserName(final UserGroupInformation ugi) { + String getQuotaUserName(final UserGroupInformation ugi) { if (userOverrideRequestAttributeKey == null) { return ugi.getShortUserName(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 7a42d0f1aa31..34fc57cb0814 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -199,7 +199,7 @@ public OperationQuota checkScanQuota(final Region region, LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " scan=" + scanRequest.getScannerId() + ": " + e.getMessage()); - rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + rsServices.getMetrics().recordThrottleException(e.getType(), quotaCache.getQuotaUserName(ugi), table.getNameAsString()); throw e; @@ -276,7 +276,7 @@ public OperationQuota checkBatchQuota(final Region region, final int numWrites, LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " numWrites=" + numWrites + " numReads=" + numReads + ": " + e.getMessage()); - rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + rsServices.getMetrics().recordThrottleException(e.getType(), quotaCache.getQuotaUserName(ugi), table.getNameAsString()); throw e; From a8599a2595ff223ecb14b30387afa2cc39dd57d6 Mon Sep 17 00:00:00 2001 From: sanjeet006py <36011005+sanjeet006py@users.noreply.github.com> Date: Sat, 4 Oct 2025 03:38:31 +0530 Subject: [PATCH 078/336] HBASE-29626: Refactor server side scan metrics for Coproc hooks (#7348) Signed-off-by: Viraj Jasani --- .../apache/hadoop/hbase/io/hfile/HFile.java | 4 +-- .../org/apache/hadoop/hbase/ipc/RpcCall.java | 4 --- .../apache/hadoop/hbase/ipc/RpcServer.java | 6 ++--- .../apache/hadoop/hbase/ipc/ServerCall.java | 11 -------- .../ThreadLocalServerSideScanMetrics.java | 23 +++++++++++++++- .../hbase/regionserver/RSRpcServices.java | 12 +++++---- .../hbase/regionserver/RegionScannerImpl.java | 26 ------------------- .../namequeues/TestNamedQueueRecorder.java | 10 ------- .../hbase/namequeues/TestRpcLogDetails.java | 10 ------- .../region/TestRegionProcedureStore.java | 10 ------- 10 files changed, 34 insertions(+), 82 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 1f8f84215488..10e2cc76c09b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; -import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.regionserver.CellSink; import org.apache.hadoop.hbase.regionserver.ShipperListener; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; @@ -190,7 +190,7 @@ public static final long getChecksumFailuresCount() { } public static final void updateReadLatency(long latencyMillis, boolean pread, boolean tooSlow) { - RpcServer.getCurrentCall().ifPresent(call -> call.updateFsReadTime(latencyMillis)); + ThreadLocalServerSideScanMetrics.addFsReadTime(latencyMillis); if (pread) { MetricsIO.getInstance().updateFsPreadTime(latencyMillis); } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java index 2d06aa7c47af..260d6e1a9803 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCall.java @@ -132,8 +132,4 @@ public interface RpcCall extends RpcCallContext { /** Returns A short string format of this call without possibly lengthy params */ String toShortString(); - - void updateFsReadTime(long latencyMillis); - - long getFsReadTime(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index bba1e66b1f93..fc6d9ea7611a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.namequeues.NamedQueueRecorder; import org.apache.hadoop.hbase.namequeues.RpcLogDetails; import org.apache.hadoop.hbase.regionserver.RSRpcServices; @@ -447,19 +448,18 @@ public Pair call(RpcCall call, MonitoredRPCHandler status) int processingTime = (int) (endTime - startTime); int qTime = (int) (startTime - receiveTime); int totalTime = (int) (endTime - receiveTime); + long fsReadTime = ThreadLocalServerSideScanMetrics.getFsReadTimeCounter().get(); if (LOG.isTraceEnabled()) { LOG.trace( "{}, response: {}, receiveTime: {}, queueTime: {}, processingTime: {}, " + "totalTime: {}, fsReadTime: {}", CurCall.get().toString(), TextFormat.shortDebugString(result), - CurCall.get().getReceiveTime(), qTime, processingTime, totalTime, - CurCall.get().getFsReadTime()); + CurCall.get().getReceiveTime(), qTime, processingTime, totalTime, fsReadTime); } // Use the raw request call size for now. long requestSize = call.getSize(); long responseSize = result.getSerializedSize(); long responseBlockSize = call.getBlockBytesScanned(); - long fsReadTime = call.getFsReadTime(); if (call.isClientCellBlockSupported()) { // Include the payload size in HBaseRpcController responseSize += call.getResponseCellSize(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java index db181d6d6f3a..8702980a10d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerCall.java @@ -102,7 +102,6 @@ public abstract class ServerCall implements RpcCa private long responseCellSize = 0; private long responseBlockSize = 0; - private long fsReadTimeMillis = 0; // cumulative size of serialized exceptions private long exceptionSize = 0; private final boolean retryImmediatelySupported; @@ -610,14 +609,4 @@ public synchronized BufferChain getResponse() { public synchronized RpcCallback getCallBack() { return this.rpcCallback; } - - @Override - public void updateFsReadTime(long latencyMillis) { - fsReadTimeMillis += latencyMillis; - } - - @Override - public long getFsReadTime() { - return fsReadTimeMillis; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java index 8c9ec24e8662..e14761ab6e18 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/ThreadLocalServerSideScanMetrics.java @@ -18,10 +18,12 @@ package org.apache.hadoop.hbase.monitoring; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; /** * Thread-local storage for server-side scan metrics that captures performance data separately for @@ -61,7 +63,8 @@ * @see RegionScanner * @see org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler */ -@InterfaceAudience.Private +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.PHOENIX) +@InterfaceStability.Evolving public final class ThreadLocalServerSideScanMetrics { private ThreadLocalServerSideScanMetrics() { } @@ -81,6 +84,9 @@ private ThreadLocalServerSideScanMetrics() { private static final ThreadLocal BLOCK_READ_OPS_COUNT = ThreadLocal.withInitial(() -> new AtomicLong(0)); + private static final ThreadLocal FS_READ_TIME = + ThreadLocal.withInitial(() -> new AtomicLong(0)); + public static void setScanMetricsEnabled(boolean enable) { IS_SCAN_METRICS_ENABLED.set(enable); } @@ -101,6 +107,10 @@ public static long addBlockReadOpsCount(long count) { return BLOCK_READ_OPS_COUNT.get().addAndGet(count); } + public static long addFsReadTime(long time) { + return FS_READ_TIME.get().addAndGet(time); + } + public static boolean isScanMetricsEnabled() { return IS_SCAN_METRICS_ENABLED.get(); } @@ -121,6 +131,10 @@ public static AtomicLong getBlockReadOpsCountCounter() { return BLOCK_READ_OPS_COUNT.get(); } + public static AtomicLong getFsReadTimeCounter() { + return FS_READ_TIME.get(); + } + public static long getBytesReadFromFsAndReset() { return getBytesReadFromFsCounter().getAndSet(0); } @@ -137,11 +151,16 @@ public static long getBlockReadOpsCountAndReset() { return getBlockReadOpsCountCounter().getAndSet(0); } + public static long getFsReadTimeAndReset() { + return getFsReadTimeCounter().getAndSet(0); + } + public static void reset() { getBytesReadFromFsAndReset(); getBytesReadFromBlockCacheAndReset(); getBytesReadFromMemstoreAndReset(); getBlockReadOpsCountAndReset(); + getFsReadTimeAndReset(); } public static void populateServerSideScanMetrics(ServerSideScanMetrics metrics) { @@ -156,5 +175,7 @@ public static void populateServerSideScanMetrics(ServerSideScanMetrics metrics) getBytesReadFromMemstoreCounter().get()); metrics.addToCounter(ServerSideScanMetrics.BLOCK_READ_OPS_COUNT_METRIC_NAME, getBlockReadOpsCountCounter().get()); + metrics.addToCounter(ServerSideScanMetrics.FS_READ_TIME_METRIC_NAME, + getFsReadTimeCounter().get()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 7bad1d99bada..e246da4bd83d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -112,6 +112,7 @@ import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterRpcServices; +import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.namequeues.NamedQueuePayload; import org.apache.hadoop.hbase.namequeues.NamedQueueRecorder; import org.apache.hadoop.hbase.namequeues.RpcLogDetails; @@ -3570,10 +3571,6 @@ private void scan(HBaseRpcController controller, ScanRequest request, RegionScan // from block size progress before writing into the response scanMetrics.setCounter(ServerSideScanMetrics.BLOCK_BYTES_SCANNED_KEY_METRIC_NAME, scannerContext.getBlockSizeProgress()); - if (rpcCall != null) { - scanMetrics.setCounter(ServerSideScanMetrics.FS_READ_TIME_METRIC_NAME, - rpcCall.getFsReadTime()); - } } } } finally { @@ -3639,6 +3636,11 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque } throw new ServiceException(e); } + boolean trackMetrics = request.hasTrackScanMetrics() && request.getTrackScanMetrics(); + ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(trackMetrics); + if (trackMetrics) { + ThreadLocalServerSideScanMetrics.reset(); + } requestCount.increment(); rpcScanRequestCount.increment(); RegionScannerContext rsx; @@ -3709,7 +3711,6 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque boolean scannerClosed = false; try { List results = new ArrayList<>(Math.min(rows, 512)); - boolean trackMetrics = request.hasTrackScanMetrics() && request.getTrackScanMetrics(); ServerSideScanMetrics scanMetrics = trackMetrics ? new ServerSideScanMetrics() : null; if (rows > 0) { boolean done = false; @@ -3791,6 +3792,7 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque scanMetrics.addToCounter(ServerSideScanMetrics.RPC_SCAN_QUEUE_WAIT_TIME_METRIC_NAME, rpcQueueWaitTime); } + ThreadLocalServerSideScanMetrics.populateServerSideScanMetrics(scanMetrics); Map metrics = scanMetrics.getMetricsMap(); ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java index 153900544d19..fe7bf89c0276 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScannerImpl.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcCallback; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState; @@ -95,8 +94,6 @@ public class RegionScannerImpl implements RegionScanner, Shipper, RpcCallback { private RegionServerServices rsServices; - private ServerSideScanMetrics scannerInitMetrics = null; - @Override public RegionInfo getRegionInfo() { return region.getRegionInfo(); @@ -147,16 +144,7 @@ private static boolean hasNonce(HRegion region, long nonce) { } finally { region.smallestReadPointCalcLock.unlock(ReadPointCalculationLock.LockType.RECORDING_LOCK); } - boolean isScanMetricsEnabled = scan.isScanMetricsEnabled(); - ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(isScanMetricsEnabled); - if (isScanMetricsEnabled) { - this.scannerInitMetrics = new ServerSideScanMetrics(); - ThreadLocalServerSideScanMetrics.reset(); - } initializeScanners(scan, additionalScanners); - if (isScanMetricsEnabled) { - ThreadLocalServerSideScanMetrics.populateServerSideScanMetrics(scannerInitMetrics); - } } public ScannerContext getContext() { @@ -289,16 +277,6 @@ public boolean nextRaw(List outResults, ScannerContext scannerContext) thr throw new UnknownScannerException("Scanner was closed"); } boolean moreValues = false; - boolean isScanMetricsEnabled = scannerContext.isTrackingMetrics(); - ThreadLocalServerSideScanMetrics.setScanMetricsEnabled(isScanMetricsEnabled); - if (isScanMetricsEnabled) { - ThreadLocalServerSideScanMetrics.reset(); - ServerSideScanMetrics scanMetrics = scannerContext.getMetrics(); - if (scannerInitMetrics != null) { - scannerInitMetrics.getMetricsMap().forEach(scanMetrics::addToCounter); - scannerInitMetrics = null; - } - } if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called // to handle scan or get operation. @@ -308,10 +286,6 @@ public boolean nextRaw(List outResults, ScannerContext scannerContext) thr moreValues = nextInternal(tmpList, scannerContext); outResults.addAll(tmpList); } - if (isScanMetricsEnabled) { - ServerSideScanMetrics scanMetrics = scannerContext.getMetrics(); - ThreadLocalServerSideScanMetrics.populateServerSideScanMetrics(scanMetrics); - } region.addReadRequestsCount(1); if (region.getMetrics() != null) { region.getMetrics().updateReadRequestCount(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java index f4cccebde03c..b08d4db191f7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestNamedQueueRecorder.java @@ -919,16 +919,6 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } - - @Override - public void updateFsReadTime(long latencyMillis) { - - } - - @Override - public long getFsReadTime() { - return 0; - } }; return rpcCall; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java index 1de0a0d31a33..bed9dea55c60 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namequeues/TestRpcLogDetails.java @@ -264,16 +264,6 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } - - @Override - public void updateFsReadTime(long latencyMillis) { - - } - - @Override - public long getFsReadTime() { - return 0; - } }; return rpcCall; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java index fdd5c7d5cf90..cd86d3424d3e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java @@ -326,16 +326,6 @@ public long getResponseExceptionSize() { @Override public void incrementResponseExceptionSize(long exceptionSize) { } - - @Override - public void updateFsReadTime(long latencyMillis) { - - } - - @Override - public long getFsReadTime() { - return 0; - } }; } } From c2e27df83d232f1a89c49c717670e8d07a3e4619 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 7 Oct 2025 17:19:32 +0800 Subject: [PATCH 079/336] HBASE-29636 Implement TimedOutTestsListener for junit 5 (#7352) Signed-off by: Chandra Sekhar K (cherry picked from commit 31520c7de69c252b5de78de58355159d596e75ac) --- .../hadoop/hbase/HBaseJupiterExtension.java | 7 +++- .../TestBuildThreadDiagnosticString.java | 39 +++++++++++++++++++ .../hadoop/hbase/TimedOutTestsListener.java | 19 ++++----- 3 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java index ff2ad14fe76b..997e3dfa357a 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java @@ -162,12 +162,17 @@ private T runWithTimeout(Invocation invocation, ExtensionContext ctx) thr } catch (ExecutionException e) { throw ExceptionUtils.throwAsUncheckedException(e.getCause()); } catch (TimeoutException e) { - + printThreadDump(); throw new JUnitException( "Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline, e); } } + private void printThreadDump() { + LOG.info("====> TEST TIMED OUT. PRINTING THREAD DUMP. <===="); + LOG.info(TimedOutTestsListener.buildThreadDiagnosticString()); + } + @Override public void interceptBeforeAllMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java new file mode 100644 index 000000000000..4071f18e2dd6 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestBuildThreadDiagnosticString.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBuildThreadDiagnosticString { + + @Test + public void test() { + String threadDump = TimedOutTestsListener.buildThreadDiagnosticString(); + System.out.println(threadDump); + assertThat(threadDump, + containsString(getClass().getName() + ".test(" + getClass().getSimpleName() + ".java:")); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java index 00860d0dde58..253d17359778 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TimedOutTestsListener.java @@ -26,9 +26,9 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.Map; import org.junit.runner.notification.Failure; @@ -40,7 +40,10 @@ */ public class TimedOutTestsListener extends RunListener { - static final String TEST_TIMED_OUT_PREFIX = "test timed out after"; + private static final String TEST_TIMED_OUT_PREFIX = "test timed out after"; + + private static final DateTimeFormatter TIMESTAMP_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS Z").withZone(ZoneId.systemDefault()); private static String INDENT = " "; @@ -67,13 +70,11 @@ public void testFailure(Failure failure) throws Exception { output.flush(); } - @SuppressWarnings("JavaUtilDate") public static String buildThreadDiagnosticString() { StringWriter sw = new StringWriter(); PrintWriter output = new PrintWriter(sw); - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); - output.println(String.format("Timestamp: %s", dateFormat.format(new Date()))); + output.println(String.format("Timestamp: %s", TIMESTAMP_FORMATTER.format(Instant.now()))); output.println(); output.println(buildThreadDump()); @@ -87,7 +88,7 @@ public static String buildThreadDiagnosticString() { return sw.toString(); } - static String buildThreadDump() { + private static String buildThreadDump() { StringBuilder dump = new StringBuilder(); Map stackTraces = Thread.getAllStackTraces(); for (Map.Entry e : stackTraces.entrySet()) { @@ -109,7 +110,7 @@ static String buildThreadDump() { return dump.toString(); } - static String buildDeadlockInfo() { + private static String buildDeadlockInfo() { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); long[] threadIds = threadBean.findMonitorDeadlockedThreads(); if (threadIds != null && threadIds.length > 0) { From 6b887db493796fd5c9c2bcd3c900186e22694488 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Wed, 8 Oct 2025 20:46:48 +0800 Subject: [PATCH 080/336] HBASE-29614 Remove static final field modification in tests around Unsafe (#7337) (#7347) Signed-off-by: Peng Lu (cherry picked from commit e0cec314c839158eac7dbd080582ebce83718a0b) --- hbase-common/pom.xml | 5 + .../hbase/util/ByteBufferUtilsTestBase.java | 600 ++++++++ .../hadoop/hbase/util/BytesTestBase.java | 578 ++++++++ .../hbase/util/TestByteBufferUtils.java | 652 +-------- .../util/TestByteBufferUtilsWoUnsafe.java | 43 + .../apache/hadoop/hbase/util/TestBytes.java | 613 +-------- .../hadoop/hbase/util/TestBytesWoUnsafe.java | 41 + hbase-server/pom.xml | 5 + .../hadoop/hbase/TestHBaseTestingUtility.java | 37 - .../hadoop/hbase/TestPortAllocator.java | 67 + .../hbase/client/FromClientSide3TestBase.java | 1208 +++++++++++++++++ .../hbase/client/TestFromClientSide3.java | 1208 +---------------- .../client/TestScannersFromClientSide.java | 2 +- .../hbase/ipc/TestProtobufRpcServiceImpl.java | 66 +- .../security/access/TestRpcAccessChecks.java | 45 +- .../util/TestFromClientSide3WoUnsafe.java | 42 +- 16 files changed, 2694 insertions(+), 2518 deletions(-) create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestPortAllocator.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/FromClientSide3TestBase.java diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 9222eca81b62..dd0c33170872 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -145,6 +145,11 @@ mockito-core test + + org.mockito + mockito-inline + test + org.slf4j jcl-over-slf4j diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java new file mode 100644 index 000000000000..194915475775 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ByteBufferUtilsTestBase.java @@ -0,0 +1,600 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.io.WritableUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ByteBufferUtilsTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(ByteBufferUtilsTestBase.class); + + private static int MAX_VLONG_LENGTH = 9; + private static Collection testNumbers; + + private byte[] array; + + @BeforeAll + public static void setUpBeforeAll() { + SortedSet a = new TreeSet<>(); + for (int i = 0; i <= 63; ++i) { + long v = -1L << i; + assertTrue(v < 0); + addNumber(a, v); + v = (1L << i) - 1; + assertTrue(v >= 0); + addNumber(a, v); + } + + testNumbers = Collections.unmodifiableSet(a); + LOG.info("Testing variable-length long serialization using: {} (count: {})", testNumbers, + testNumbers.size()); + assertEquals(1753, testNumbers.size()); + assertEquals(Long.MIN_VALUE, a.first().longValue()); + assertEquals(Long.MAX_VALUE, a.last().longValue()); + } + + /** + * Create an array with sample data. + */ + @BeforeEach + public void setUp() { + array = new byte[8]; + for (int i = 0; i < array.length; ++i) { + array[i] = (byte) ('a' + i); + } + } + + private static void addNumber(Set a, long l) { + if (l != Long.MIN_VALUE) { + a.add(l - 1); + } + a.add(l); + if (l != Long.MAX_VALUE) { + a.add(l + 1); + } + for (long divisor = 3; divisor <= 10; ++divisor) { + for (long delta = -1; delta <= 1; ++delta) { + a.add(l / divisor + delta); + } + } + } + + @Test + public void testReadWriteVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + } + } + + @Test + public void testReadWriteConsecutiveVLong() { + for (long l : testNumbers) { + ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + ByteBufferUtils.writeVLong(b, l - 4); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(b)); + assertEquals(l - 4, ByteBufferUtils.readVLong(b)); + b.flip(); + assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); + } + } + + @Test + public void testConsistencyWithHadoopVLong() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + for (long l : testNumbers) { + baos.reset(); + ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); + ByteBufferUtils.writeVLong(b, l); + String bufStr = Bytes.toStringBinary(b.array(), b.arrayOffset(), b.position()); + WritableUtils.writeVLong(dos, l); + String baosStr = Bytes.toStringBinary(baos.toByteArray()); + assertEquals(baosStr, bufStr); + } + } + + /** + * Test copying to stream from buffer. + */ + @Test + public void testMoveBufferToStream() throws IOException { + final int arrayOffset = 7; + final int initialPosition = 10; + final int endPadding = 5; + byte[] arrayWrapper = new byte[arrayOffset + initialPosition + array.length + endPadding]; + System.arraycopy(array, 0, arrayWrapper, arrayOffset + initialPosition, array.length); + ByteBuffer buffer = + ByteBuffer.wrap(arrayWrapper, arrayOffset, initialPosition + array.length).slice(); + assertEquals(initialPosition + array.length, buffer.limit()); + assertEquals(0, buffer.position()); + buffer.position(initialPosition); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteBufferUtils.moveBufferToStream(bos, buffer, array.length); + assertArrayEquals(array, bos.toByteArray()); + assertEquals(initialPosition + array.length, buffer.position()); + } + + /** + * Test copying to stream from buffer with offset. + * @throws IOException On test failure. + */ + @Test + public void testCopyToStreamWithOffset() throws IOException { + ByteBuffer buffer = ByteBuffer.wrap(array); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + + ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2, array.length / 2); + + byte[] returnedArray = bos.toByteArray(); + for (int i = 0; i < array.length / 2; ++i) { + int pos = array.length / 2 + i; + assertEquals(returnedArray[i], array[pos]); + } + } + + /** + * Test copying data from stream. + * @throws IOException On test failure. + */ + @Test + public void testCopyFromStream() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(array.length); + ByteArrayInputStream bis = new ByteArrayInputStream(array); + DataInputStream dis = new DataInputStream(bis); + + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2); + ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length - array.length / 2); + for (int i = 0; i < array.length; ++i) { + assertEquals(array[i], buffer.get(i)); + } + } + + /** + * Test copying from buffer. + */ + @Test + public void testCopyFromBuffer() { + ByteBuffer srcBuffer = ByteBuffer.allocate(array.length); + ByteBuffer dstBuffer = ByteBuffer.allocate(array.length); + srcBuffer.put(array); + + ByteBufferUtils.copyFromBufferToBuffer(srcBuffer, dstBuffer, array.length / 2, + array.length / 4); + for (int i = 0; i < array.length / 4; ++i) { + assertEquals(srcBuffer.get(i + array.length / 2), dstBuffer.get(i)); + } + } + + /** + * Test 7-bit encoding of integers. + * @throws IOException On test failure. + */ + @Test + public void testCompressedInt() throws IOException { + testCompressedInt(0); + testCompressedInt(Integer.MAX_VALUE); + testCompressedInt(Integer.MIN_VALUE); + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testCompressedInt((128 << i)); + } + } + + /** + * Test how much bytes we need to store integer. + */ + @Test + public void testIntFitsIn() { + assertEquals(1, ByteBufferUtils.intFitsIn(0)); + assertEquals(1, ByteBufferUtils.intFitsIn(1)); + assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8)); + assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16)); + assertEquals(4, ByteBufferUtils.intFitsIn(-1)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE)); + assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE)); + } + + /** + * Test how much bytes we need to store long. + */ + @Test + public void testLongFitsIn() { + assertEquals(1, ByteBufferUtils.longFitsIn(0)); + assertEquals(1, ByteBufferUtils.longFitsIn(1)); + assertEquals(3, ByteBufferUtils.longFitsIn(1L << 16)); + assertEquals(5, ByteBufferUtils.longFitsIn(1L << 32)); + assertEquals(8, ByteBufferUtils.longFitsIn(-1)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE)); + assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE)); + } + + /** + * Test if we are comparing equal bytes. + */ + @Test + public void testArePartEqual() { + byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 }; + ByteBuffer buffer = ByteBuffer.wrap(array); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4)); + assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2)); + assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3)); + } + + /** + * Test serializing int to bytes + */ + @Test + public void testPutInt() { + testPutInt(0); + testPutInt(Integer.MAX_VALUE); + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i) - 1); + } + + for (int i = 0; i < 3; i++) { + testPutInt((128 << i)); + } + } + + @Test + public void testToBytes() { + ByteBuffer buffer = ByteBuffer.allocate(5); + buffer.put(new byte[] { 0, 1, 2, 3, 4 }); + assertEquals(5, buffer.position()); + assertEquals(5, buffer.limit()); + byte[] copy = ByteBufferUtils.toBytes(buffer, 2); + assertArrayEquals(new byte[] { 2, 3, 4 }, copy); + assertEquals(5, buffer.position()); + assertEquals(5, buffer.limit()); + } + + @Test + public void testToPrimitiveTypes() { + ByteBuffer buffer = ByteBuffer.allocate(15); + long l = 988L; + int i = 135; + short s = 7; + buffer.putLong(l); + buffer.putShort(s); + buffer.putInt(i); + assertEquals(l, ByteBufferUtils.toLong(buffer, 0)); + assertEquals(s, ByteBufferUtils.toShort(buffer, 8)); + assertEquals(i, ByteBufferUtils.toInt(buffer, 10)); + } + + @Test + public void testCopyFromArrayToBuffer() { + byte[] b = new byte[15]; + b[0] = -1; + long l = 988L; + int i = 135; + short s = 7; + Bytes.putLong(b, 1, l); + Bytes.putShort(b, 9, s); + Bytes.putInt(b, 11, i); + ByteBuffer buffer = ByteBuffer.allocate(14); + ByteBufferUtils.copyFromArrayToBuffer(buffer, b, 1, 14); + buffer.rewind(); + assertEquals(l, buffer.getLong()); + assertEquals(s, buffer.getShort()); + assertEquals(i, buffer.getInt()); + } + + private void testCopyFromSrcToDestWithThreads(Object input, Object output, List lengthes, + List offsets) throws InterruptedException { + assertTrue((input instanceof ByteBuffer) || (input instanceof byte[])); + assertTrue((output instanceof ByteBuffer) || (output instanceof byte[])); + assertEquals(lengthes.size(), offsets.size()); + + final int threads = lengthes.size(); + CountDownLatch latch = new CountDownLatch(1); + List exes = new ArrayList<>(threads); + int oldInputPos = (input instanceof ByteBuffer) ? ((ByteBuffer) input).position() : 0; + int oldOutputPos = (output instanceof ByteBuffer) ? ((ByteBuffer) output).position() : 0; + for (int i = 0; i != threads; ++i) { + int offset = offsets.get(i); + int length = lengthes.get(i); + exes.add(() -> { + try { + latch.await(); + if (input instanceof ByteBuffer && output instanceof byte[]) { + ByteBufferUtils.copyFromBufferToArray((byte[]) output, (ByteBuffer) input, offset, + offset, length); + } + if (input instanceof byte[] && output instanceof ByteBuffer) { + ByteBufferUtils.copyFromArrayToBuffer((ByteBuffer) output, offset, (byte[]) input, + offset, length); + } + if (input instanceof ByteBuffer && output instanceof ByteBuffer) { + ByteBufferUtils.copyFromBufferToBuffer((ByteBuffer) input, (ByteBuffer) output, offset, + offset, length); + } + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + }); + } + ExecutorService service = Executors.newFixedThreadPool(threads); + exes.forEach(service::execute); + latch.countDown(); + service.shutdown(); + assertTrue(service.awaitTermination(5, TimeUnit.SECONDS)); + if (input instanceof ByteBuffer) { + assertEquals(oldInputPos, ((ByteBuffer) input).position()); + } + if (output instanceof ByteBuffer) { + assertEquals(oldOutputPos, ((ByteBuffer) output).position()); + } + String inputString = (input instanceof ByteBuffer) + ? Bytes.toString(Bytes.toBytes((ByteBuffer) input)) + : Bytes.toString((byte[]) input); + String outputString = (output instanceof ByteBuffer) + ? Bytes.toString(Bytes.toBytes((ByteBuffer) output)) + : Bytes.toString((byte[]) output); + assertEquals(inputString, outputString); + } + + @Test + public void testCopyFromSrcToDestWithThreads() throws InterruptedException { + List words = + Arrays.asList(Bytes.toBytes("with"), Bytes.toBytes("great"), Bytes.toBytes("power"), + Bytes.toBytes("comes"), Bytes.toBytes("great"), Bytes.toBytes("responsibility")); + List lengthes = words.stream().map(v -> v.length).collect(Collectors.toList()); + List offsets = new ArrayList<>(words.size()); + for (int i = 0; i != words.size(); ++i) { + offsets.add(words.subList(0, i).stream().mapToInt(v -> v.length).sum()); + } + + int totalSize = words.stream().mapToInt(v -> v.length).sum(); + byte[] fullContent = new byte[totalSize]; + int offset = 0; + for (byte[] w : words) { + offset = Bytes.putBytes(fullContent, offset, w, 0, w.length); + } + + // test copyFromBufferToArray + for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + words.forEach(input::put); + byte[] output = new byte[totalSize]; + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + + // test copyFromArrayToBuffer + for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + byte[] input = fullContent; + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + + // test copyFromBufferToBuffer + for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + words.forEach(input::put); + for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), + ByteBuffer.allocate(totalSize))) { + testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); + } + } + } + + @Test + public void testCopyFromBufferToArray() { + ByteBuffer buffer = ByteBuffer.allocate(15); + buffer.put((byte) -1); + long l = 988L; + int i = 135; + short s = 7; + buffer.putShort(s); + buffer.putInt(i); + buffer.putLong(l); + byte[] b = new byte[15]; + ByteBufferUtils.copyFromBufferToArray(b, buffer, 1, 1, 14); + assertEquals(s, Bytes.toShort(b, 1)); + assertEquals(i, Bytes.toInt(b, 3)); + assertEquals(l, Bytes.toLong(b, 7)); + } + + @Test + public void testRelativeCopyFromBuffertoBuffer() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + fillBB(bb1, (byte) 5); + ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); + assertTrue(bb1.position() == bb2.position()); + assertTrue(bb1.limit() == bb2.limit()); + bb1 = ByteBuffer.allocateDirect(135); + bb2 = ByteBuffer.allocateDirect(135); + fillBB(bb1, (byte) 5); + ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); + assertTrue(bb1.position() == bb2.position()); + assertTrue(bb1.limit() == bb2.limit()); + } + + @Test + public void testCompareTo() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + byte[] b = new byte[71]; + fillBB(bb1, (byte) 5); + fillBB(bb2, (byte) 5); + fillArray(b, (byte) 5); + assertEquals(0, ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), b, 0, b.length) > 0); + bb2.put(134, (byte) 6); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) < 0); + bb2.put(6, (byte) 4); + assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) > 0); + // Assert reverse comparing BB and bytearray works. + ByteBuffer bb3 = ByteBuffer.allocate(135); + fillBB(bb3, (byte) 0); + byte[] b3 = new byte[135]; + fillArray(b3, (byte) 1); + int result = ByteBufferUtils.compareTo(b3, 0, b3.length, bb3, 0, bb3.remaining()); + assertTrue(result > 0); + result = ByteBufferUtils.compareTo(bb3, 0, bb3.remaining(), b3, 0, b3.length); + assertTrue(result < 0); + byte[] b4 = Bytes.toBytes("123"); + ByteBuffer bb4 = ByteBuffer.allocate(10 + b4.length); + for (int i = 10; i < bb4.capacity(); ++i) { + bb4.put(i, b4[i - 10]); + } + result = ByteBufferUtils.compareTo(b4, 0, b4.length, bb4, 10, b4.length); + assertEquals(0, result); + } + + @Test + public void testEquals() { + byte[] a = Bytes.toBytes("http://A"); + ByteBuffer bb = ByteBuffer.wrap(a); + + assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, + HConstants.EMPTY_BYTE_BUFFER, 0, 0)); + + assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, bb, 0, a.length)); + + assertFalse(ByteBufferUtils.equals(bb, 0, 0, HConstants.EMPTY_BYTE_BUFFER, 0, a.length)); + + assertTrue(ByteBufferUtils.equals(bb, 0, a.length, bb, 0, a.length)); + + assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, + HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + + assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, a, 0, a.length)); + + assertFalse(ByteBufferUtils.equals(bb, 0, a.length, HConstants.EMPTY_BYTE_ARRAY, 0, 0)); + + assertTrue(ByteBufferUtils.equals(bb, 0, a.length, a, 0, a.length)); + } + + @Test + public void testFindCommonPrefix() { + ByteBuffer bb1 = ByteBuffer.allocate(135); + ByteBuffer bb2 = ByteBuffer.allocate(135); + ByteBuffer bb3 = ByteBuffer.allocateDirect(135); + byte[] b = new byte[71]; + + fillBB(bb1, (byte) 5); + fillBB(bb2, (byte) 5); + fillBB(bb3, (byte) 5); + fillArray(b, (byte) 5); + + assertEquals(135, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + assertEquals(71, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); + assertEquals(135, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb3, 0, bb3.remaining())); + assertEquals(71, ByteBufferUtils.findCommonPrefix(bb3, 0, bb3.remaining(), b, 0, b.length)); + + b[13] = 9; + assertEquals(13, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); + + bb2.put(134, (byte) 6); + assertEquals(134, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + + bb2.put(6, (byte) 4); + assertEquals(6, + ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); + } + + // Below are utility methods invoked from test methods + private static void testCompressedInt(int value) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteBufferUtils.putCompressedInt(bos, value); + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + int parsedValue = ByteBufferUtils.readCompressedInt(bis); + assertEquals(value, parsedValue); + } + + private static void testPutInt(int value) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + ByteBufferUtils.putInt(baos, value); + } catch (IOException e) { + throw new RuntimeException("Bug in putIn()", e); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + try { + assertEquals(dis.readInt(), value); + } catch (IOException e) { + throw new RuntimeException("Bug in test!", e); + } + } + + private static void fillBB(ByteBuffer bb, byte b) { + for (int i = bb.position(); i < bb.limit(); i++) { + bb.put(i, b); + } + } + + private static void fillArray(byte[] bb, byte b) { + for (int i = 0; i < bb.length; i++) { + bb[i] = b; + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java new file mode 100644 index 000000000000..96df8bc39396 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/BytesTestBase.java @@ -0,0 +1,578 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.hadoop.io.WritableUtils; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BytesTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(BytesTestBase.class); + + @Test + public void testShort() throws Exception { + for (short n : Arrays.asList(Short.MIN_VALUE, (short) -100, (short) -1, (short) 0, (short) 1, + (short) 300, Short.MAX_VALUE)) { + byte[] bytes = Bytes.toBytes(n); + assertEquals(Bytes.toShort(bytes, 0, bytes.length), n); + } + } + + @Test + public void testNullHashCode() { + byte[] b = null; + Exception ee = null; + try { + Bytes.hashCode(b); + } catch (Exception e) { + ee = e; + } + assertNotNull(ee); + } + + @Test + public void testAdd() { + byte[] a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + byte[] b = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte[] c = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }; + byte[] result1 = Bytes.add(a, b, c); + byte[] result2 = Bytes.add(new byte[][] { a, b, c }); + assertEquals(0, Bytes.compareTo(result1, result2)); + } + + @Test + public void testSplit() { + byte[] lowest = Bytes.toBytes("AAA"); + byte[] middle = Bytes.toBytes("CCC"); + byte[] highest = Bytes.toBytes("EEE"); + byte[][] parts = Bytes.split(lowest, highest, 1); + for (byte[] bytes : parts) { + LOG.info(Bytes.toString(bytes)); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + // Now divide into three parts. Change highest so split is even. + highest = Bytes.toBytes("DDD"); + parts = Bytes.split(lowest, highest, 2); + for (byte[] part : parts) { + LOG.info(Bytes.toString(part)); + } + assertEquals(4, parts.length); + // Assert that 3rd part is 'CCC'. + assertTrue(Bytes.equals(parts[2], middle)); + } + + @Test + public void testSplit2() { + // More split tests. + byte[] lowest = Bytes.toBytes("http://A"); + byte[] highest = Bytes.toBytes("http://z"); + byte[] middle = Bytes.toBytes("http://]"); + byte[][] parts = Bytes.split(lowest, highest, 1); + for (byte[] part : parts) { + LOG.info(Bytes.toString(part)); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + } + + @Test + public void testSplit3() { + // Test invalid split cases + byte[] low = { 1, 1, 1 }; + byte[] high = { 1, 1, 3 }; + + // If swapped, should throw IAE + try { + Bytes.split(high, low, 1); + fail("Should not be able to split if low > high"); + } catch (IllegalArgumentException iae) { + // Correct + } + + // Single split should work + byte[][] parts = Bytes.split(low, high, 1); + for (int i = 0; i < parts.length; i++) { + LOG.info("" + i + " -> " + Bytes.toStringBinary(parts[i])); + } + assertEquals(3, parts.length, "Returned split should have 3 parts but has " + parts.length); + + // If split more than once, use additional byte to split + parts = Bytes.split(low, high, 2); + assertNotNull(parts, "Split with an additional byte"); + assertEquals(parts.length, low.length + 1); + + // Split 0 times should throw IAE + try { + Bytes.split(low, high, 0); + fail("Should not be able to split 0 times"); + } catch (IllegalArgumentException iae) { + // Correct + } + } + + @Test + public void testToInt() { + int[] ints = { -1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE }; + for (int anInt : ints) { + byte[] b = Bytes.toBytes(anInt); + assertEquals(anInt, Bytes.toInt(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(anInt, Bytes.toInt(b2, 1)); + assertEquals(anInt, Bytes.toInt(b2, 1, Bytes.SIZEOF_INT)); + } + } + + @Test + public void testToLong() { + long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; + for (long aLong : longs) { + byte[] b = Bytes.toBytes(aLong); + assertEquals(aLong, Bytes.toLong(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(aLong, Bytes.toLong(b2, 1)); + assertEquals(aLong, Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG)); + } + } + + @Test + public void testToFloat() { + float[] floats = { -1f, 123.123f, Float.MAX_VALUE }; + for (float aFloat : floats) { + byte[] b = Bytes.toBytes(aFloat); + assertEquals(aFloat, Bytes.toFloat(b), 0.0f); + byte[] b2 = bytesWithOffset(b); + assertEquals(aFloat, Bytes.toFloat(b2, 1), 0.0f); + } + } + + @Test + public void testToDouble() { + double[] doubles = { Double.MIN_VALUE, Double.MAX_VALUE }; + for (double aDouble : doubles) { + byte[] b = Bytes.toBytes(aDouble); + assertEquals(aDouble, Bytes.toDouble(b), 0.0); + byte[] b2 = bytesWithOffset(b); + assertEquals(aDouble, Bytes.toDouble(b2, 1), 0.0); + } + } + + @Test + public void testToBigDecimal() { + BigDecimal[] decimals = + { new BigDecimal("-1"), new BigDecimal("123.123"), new BigDecimal("123123123123") }; + for (BigDecimal decimal : decimals) { + byte[] b = Bytes.toBytes(decimal); + assertEquals(decimal, Bytes.toBigDecimal(b)); + byte[] b2 = bytesWithOffset(b); + assertEquals(decimal, Bytes.toBigDecimal(b2, 1, b.length)); + } + } + + private byte[] bytesWithOffset(byte[] src) { + // add one byte in front to test offset + byte[] result = new byte[src.length + 1]; + result[0] = (byte) 0xAA; + System.arraycopy(src, 0, result, 1, src.length); + return result; + } + + @Test + public void testToBytesForByteBuffer() { + byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ByteBuffer target = ByteBuffer.wrap(array); + target.position(2); + target.limit(7); + + byte[] actual = Bytes.toBytes(target); + byte[] expected = { 0, 1, 2, 3, 4, 5, 6 }; + assertArrayEquals(expected, actual); + assertEquals(2, target.position()); + assertEquals(7, target.limit()); + + ByteBuffer target2 = target.slice(); + assertEquals(0, target2.position()); + assertEquals(5, target2.limit()); + + byte[] actual2 = Bytes.toBytes(target2); + byte[] expected2 = { 2, 3, 4, 5, 6 }; + assertArrayEquals(expected2, actual2); + assertEquals(0, target2.position()); + assertEquals(5, target2.limit()); + } + + @Test + public void testGetBytesForByteBuffer() { + byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + ByteBuffer target = ByteBuffer.wrap(array); + target.position(2); + target.limit(7); + + byte[] actual = Bytes.getBytes(target); + byte[] expected = { 2, 3, 4, 5, 6 }; + assertArrayEquals(expected, actual); + assertEquals(2, target.position()); + assertEquals(7, target.limit()); + } + + @Test + public void testReadAsVLong() throws Exception { + long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; + for (long aLong : longs) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(baos); + WritableUtils.writeVLong(output, aLong); + byte[] long_bytes_no_offset = baos.toByteArray(); + assertEquals(aLong, Bytes.readAsVLong(long_bytes_no_offset, 0)); + byte[] long_bytes_with_offset = bytesWithOffset(long_bytes_no_offset); + assertEquals(aLong, Bytes.readAsVLong(long_bytes_with_offset, 1)); + } + } + + @Test + public void testToStringBinaryForBytes() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + String actual = Bytes.toStringBinary(array); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + + String actual2 = Bytes.toStringBinary(array, 2, 3); + String expected2 = "azA"; + assertEquals(expected2, actual2); + } + + @Test + public void testToStringBinaryForArrayBasedByteBuffer() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + ByteBuffer target = ByteBuffer.wrap(array); + String actual = Bytes.toStringBinary(target); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + } + + @Test + public void testToStringBinaryForReadOnlyByteBuffer() { + byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; + ByteBuffer target = ByteBuffer.wrap(array).asReadOnlyBuffer(); + String actual = Bytes.toStringBinary(target); + String expected = "09azAZ@\\x01"; + assertEquals(expected, actual); + } + + @Test + public void testBinarySearch() { + byte[][] arr = { { 1 }, { 3 }, { 5 }, { 7 }, { 9 }, { 11 }, { 13 }, { 15 }, }; + byte[] key1 = { 3, 1 }; + byte[] key2 = { 4, 9 }; + byte[] key2_2 = { 4 }; + byte[] key3 = { 5, 11 }; + byte[] key4 = { 0 }; + byte[] key5 = { 2 }; + + assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1)); + assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1)); + assertEquals(-(2 + 1), Arrays.binarySearch(arr, key2_2, Bytes.BYTES_COMPARATOR)); + assertEquals(-(2 + 1), Bytes.binarySearch(arr, key2, 0, 1)); + assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1)); + assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1)); + assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1)); + assertEquals(-1, Bytes.binarySearch(arr, key4, 0, 1)); + assertEquals(-2, Bytes.binarySearch(arr, key5, 0, 1)); + + // Search for values to the left and to the right of each item in the array. + for (int i = 0; i < arr.length; ++i) { + assertEquals(-(i + 1), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] - 1) }, 0, 1)); + assertEquals(-(i + 2), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] + 1) }, 0, 1)); + } + } + + @Test + public void testToStringBytesBinaryReversible() { + byte[] randomBytes = new byte[1000]; + for (int i = 0; i < 1000; i++) { + Bytes.random(randomBytes); + verifyReversibleForBytes(randomBytes); + } + // some specific cases + verifyReversibleForBytes(new byte[] {}); + verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D' }); + verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D', '\\' }); + } + + private void verifyReversibleForBytes(byte[] originalBytes) { + String convertedString = Bytes.toStringBinary(originalBytes); + byte[] convertedBytes = Bytes.toBytesBinary(convertedString); + if (Bytes.compareTo(originalBytes, convertedBytes) != 0) { + fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) + ",\nStringBinary: " + + convertedString); + } + } + + @Test + public void testStartsWith() { + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h"))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes(""))); + assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello"))); + assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld"))); + assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello"))); + } + + @Test + public void testIncrementBytes() { + assertTrue(checkTestIncrementBytes(10, 1)); + assertTrue(checkTestIncrementBytes(12, 123435445)); + assertTrue(checkTestIncrementBytes(124634654, 1)); + assertTrue(checkTestIncrementBytes(10005460, 5005645)); + assertTrue(checkTestIncrementBytes(1, -1)); + assertTrue(checkTestIncrementBytes(10, -1)); + assertTrue(checkTestIncrementBytes(10, -5)); + assertTrue(checkTestIncrementBytes(1005435000, -5)); + assertTrue(checkTestIncrementBytes(10, -43657655)); + assertTrue(checkTestIncrementBytes(-1, 1)); + assertTrue(checkTestIncrementBytes(-26, 5034520)); + assertTrue(checkTestIncrementBytes(-10657200, 5)); + assertTrue(checkTestIncrementBytes(-12343250, 45376475)); + assertTrue(checkTestIncrementBytes(-10, -5)); + assertTrue(checkTestIncrementBytes(-12343250, -5)); + assertTrue(checkTestIncrementBytes(-12, -34565445)); + assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); + } + + private static boolean checkTestIncrementBytes(long val, long amount) { + byte[] value = Bytes.toBytes(val); + byte[] testValue = { -1, -1, -1, -1, -1, -1, -1, -1 }; + if (value[0] > 0) { + testValue = new byte[Bytes.SIZEOF_LONG]; + } + System.arraycopy(value, 0, testValue, testValue.length - value.length, value.length); + + long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); + + return (Bytes.toLong(testValue) + amount) == incrementResult; + } + + @Test + public void testFixedSizeString() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + Bytes.writeStringFixedSize(dos, "Hello", 5); + Bytes.writeStringFixedSize(dos, "World", 18); + Bytes.writeStringFixedSize(dos, "", 9); + + try { + // Use a long dash which is three bytes in UTF-8. If encoding happens + // using ISO-8859-1, this will fail. + Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9); + fail("Exception expected"); + } catch (IOException ex) { + assertEquals( + "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " + "length 9", + ex.getMessage()); + } + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataInputStream dis = new DataInputStream(bais); + assertEquals("Hello", Bytes.readStringFixedSize(dis, 5)); + assertEquals("World", Bytes.readStringFixedSize(dis, 18)); + assertEquals("", Bytes.readStringFixedSize(dis, 9)); + } + + @Test + public void testCopy() { + byte[] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + byte[] copy = Bytes.copy(bytes); + assertNotSame(bytes, copy); + assertTrue(Bytes.equals(bytes, copy)); + } + + @Test + public void testToBytesBinaryTrailingBackslashes() { + try { + Bytes.toBytesBinary("abc\\x00\\x01\\"); + } catch (StringIndexOutOfBoundsException ex) { + fail("Illegal string access: " + ex.getMessage()); + } + } + + @Test + public void testToStringBinary_toBytesBinary_Reversable() { + String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17)); + assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0); + } + + @Test + public void testUnsignedBinarySearch() { + byte[] bytes = new byte[] { 0, 5, 123, 127, -128, -100, -1 }; + assertEquals(1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 5)); + assertEquals(3, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 127)); + assertEquals(4, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -128)); + assertEquals(5, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -100)); + assertEquals(6, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -1)); + assertEquals(-1 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 2)); + assertEquals(-6 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -5)); + } + + @Test + public void testUnsignedIncrement() { + byte[] a = Bytes.toBytes(0); + int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0); + assertEquals(1, a2); + + byte[] b = Bytes.toBytes(-1); + byte[] actual = Bytes.unsignedCopyAndIncrement(b); + assertNotSame(b, actual); + byte[] expected = new byte[] { 1, 0, 0, 0, 0 }; + assertArrayEquals(expected, actual); + + byte[] c = Bytes.toBytes(255);// should wrap to the next significant byte + int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0); + assertEquals(256, c2); + } + + @Test + public void testIndexOf() { + byte[] array = Bytes.toBytes("hello"); + assertEquals(1, Bytes.indexOf(array, (byte) 'e')); + assertEquals(4, Bytes.indexOf(array, (byte) 'o')); + assertEquals(-1, Bytes.indexOf(array, (byte) 'a')); + assertEquals(0, Bytes.indexOf(array, Bytes.toBytes("hel"))); + assertEquals(2, Bytes.indexOf(array, Bytes.toBytes("ll"))); + assertEquals(-1, Bytes.indexOf(array, Bytes.toBytes("hll"))); + } + + @Test + public void testContains() { + byte[] array = Bytes.toBytes("hello world"); + assertTrue(Bytes.contains(array, (byte) 'e')); + assertTrue(Bytes.contains(array, (byte) 'd')); + assertFalse(Bytes.contains(array, (byte) 'a')); + assertTrue(Bytes.contains(array, Bytes.toBytes("world"))); + assertTrue(Bytes.contains(array, Bytes.toBytes("ello"))); + assertFalse(Bytes.contains(array, Bytes.toBytes("owo"))); + } + + @Test + public void testZero() { + byte[] array = Bytes.toBytes("hello"); + Bytes.zero(array); + for (byte b : array) { + assertEquals(0, b); + } + array = Bytes.toBytes("hello world"); + Bytes.zero(array, 2, 7); + assertFalse(array[0] == 0); + assertFalse(array[1] == 0); + for (int i = 2; i < 9; i++) { + assertEquals(0, array[i]); + } + for (int i = 9; i < array.length; i++) { + assertFalse(array[i] == 0); + } + } + + @Test + public void testPutBuffer() { + byte[] b = new byte[100]; + for (byte i = 0; i < 100; i++) { + Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[] { i })); + } + for (byte i = 0; i < 100; i++) { + assertEquals(i, b[i]); + } + } + + @Test + public void testToFromHex() { + List testStrings = new ArrayList<>(8); + testStrings.addAll(Arrays.asList("", "00", "A0", "ff", "FFffFFFFFFFFFF", "12", + "0123456789abcdef", "283462839463924623984692834692346ABCDFEDDCA0")); + for (String testString : testStrings) { + byte[] byteData = Bytes.fromHex(testString); + assertEquals(testString.length() / 2, byteData.length); + String result = Bytes.toHex(byteData); + assertTrue(testString.equalsIgnoreCase(result)); + } + + List testByteData = new ArrayList<>(5); + testByteData.addAll(Arrays.asList(new byte[0], new byte[1], new byte[10], + new byte[] { 1, 2, 3, 4, 5 }, new byte[] { (byte) 0xFF })); + Random rand = ThreadLocalRandom.current(); + for (int i = 0; i < 20; i++) { + byte[] bytes = new byte[rand.nextInt(100)]; + Bytes.random(bytes); + testByteData.add(bytes); + } + + for (byte[] testData : testByteData) { + String hexString = Bytes.toHex(testData); + assertEquals(testData.length * 2, hexString.length()); + byte[] result = Bytes.fromHex(hexString); + assertArrayEquals(testData, result); + } + } + + @Test + public void testFindCommonPrefix() throws Exception { + // tests for common prefixes less than 8 bytes in length (i.e. using non-vectorized path) + byte[] hello = Bytes.toBytes("hello"); + byte[] helloWorld = Bytes.toBytes("helloworld"); + + assertEquals(5, + Bytes.findCommonPrefix(hello, helloWorld, hello.length, helloWorld.length, 0, 0)); + assertEquals(5, Bytes.findCommonPrefix(hello, hello, hello.length, hello.length, 0, 0)); + assertEquals(3, Bytes.findCommonPrefix(hello, hello, hello.length - 2, hello.length - 2, 2, 2)); + assertEquals(0, Bytes.findCommonPrefix(hello, hello, 0, 0, 0, 0)); + + // tests for common prefixes greater than 8 bytes in length which may use the vectorized path + byte[] hellohello = Bytes.toBytes("hellohello"); + byte[] hellohellohi = Bytes.toBytes("hellohellohi"); + + assertEquals(10, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, + hellohellohi.length, 0, 0)); + assertEquals(10, Bytes.findCommonPrefix(hellohellohi, hellohello, hellohellohi.length, + hellohello.length, 0, 0)); + assertEquals(10, + Bytes.findCommonPrefix(hellohello, hellohello, hellohello.length, hellohello.length, 0, 0)); + + hellohello[2] = 0; + assertEquals(2, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, + hellohellohi.length, 0, 0)); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java index 8f180d266c2c..b451d71ed879 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java @@ -17,654 +17,12 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.nio.ByteBuff; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; -import org.apache.hadoop.io.WritableUtils; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@Category({ MiscTests.class, MediumTests.class }) -@RunWith(Parameterized.class) -public class TestByteBufferUtils { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestByteBufferUtils.class); - - private static final String UNSAFE_AVAIL_NAME = "UNSAFE_AVAIL"; - private static final String UNSAFE_UNALIGNED_NAME = "UNSAFE_UNALIGNED"; - private byte[] array; - - @AfterClass - public static void afterClass() throws Exception { - detectAvailabilityOfUnsafe(); - } - - @Parameterized.Parameters - public static Collection parameters() { - return HBaseCommonTestingUtility.BOOLEAN_PARAMETERIZED; - } - - private static void setUnsafe(String fieldName, boolean value) throws Exception { - Field field = ByteBufferUtils.class.getDeclaredField(fieldName); - field.setAccessible(true); - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - int oldModifiers = field.getModifiers(); - modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL); - try { - field.set(null, value); - } finally { - modifiersField.setInt(field, oldModifiers); - } - } - - static void disableUnsafe() throws Exception { - if (ByteBufferUtils.UNSAFE_AVAIL) { - setUnsafe(UNSAFE_AVAIL_NAME, false); - } - if (ByteBufferUtils.UNSAFE_UNALIGNED) { - setUnsafe(UNSAFE_UNALIGNED_NAME, false); - } - assertFalse(ByteBufferUtils.UNSAFE_AVAIL); - assertFalse(ByteBufferUtils.UNSAFE_UNALIGNED); - } - - static void detectAvailabilityOfUnsafe() throws Exception { - if (ByteBufferUtils.UNSAFE_AVAIL != HBasePlatformDependent.isUnsafeAvailable()) { - setUnsafe(UNSAFE_AVAIL_NAME, HBasePlatformDependent.isUnsafeAvailable()); - } - if (ByteBufferUtils.UNSAFE_UNALIGNED != HBasePlatformDependent.unaligned()) { - setUnsafe(UNSAFE_UNALIGNED_NAME, HBasePlatformDependent.unaligned()); - } - assertEquals(ByteBufferUtils.UNSAFE_AVAIL, HBasePlatformDependent.isUnsafeAvailable()); - assertEquals(ByteBufferUtils.UNSAFE_UNALIGNED, HBasePlatformDependent.unaligned()); - } - - public TestByteBufferUtils(boolean useUnsafeIfPossible) throws Exception { - if (useUnsafeIfPossible) { - detectAvailabilityOfUnsafe(); - } else { - disableUnsafe(); - } - } - - /** - * Create an array with sample data. - */ - @Before - public void setUp() { - array = new byte[8]; - for (int i = 0; i < array.length; ++i) { - array[i] = (byte) ('a' + i); - } - } - - private static final int MAX_VLONG_LENGTH = 9; - private static final Collection testNumbers; - - private static void addNumber(Set a, long l) { - if (l != Long.MIN_VALUE) { - a.add(l - 1); - } - a.add(l); - if (l != Long.MAX_VALUE) { - a.add(l + 1); - } - for (long divisor = 3; divisor <= 10; ++divisor) { - for (long delta = -1; delta <= 1; ++delta) { - a.add(l / divisor + delta); - } - } - } - - static { - SortedSet a = new TreeSet<>(); - for (int i = 0; i <= 63; ++i) { - long v = -1L << i; - assertTrue(v < 0); - addNumber(a, v); - v = (1L << i) - 1; - assertTrue(v >= 0); - addNumber(a, v); - } - - testNumbers = Collections.unmodifiableSet(a); - System.err.println("Testing variable-length long serialization using: " + testNumbers - + " (count: " + testNumbers.size() + ")"); - assertEquals(1753, testNumbers.size()); - assertEquals(Long.MIN_VALUE, a.first().longValue()); - assertEquals(Long.MAX_VALUE, a.last().longValue()); - } - - @Test - public void testReadWriteVLong() { - for (long l : testNumbers) { - ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(b)); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - } - } - - @Test - public void testReadWriteConsecutiveVLong() { - for (long l : testNumbers) { - ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - ByteBufferUtils.writeVLong(b, l - 4); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(b)); - assertEquals(l - 4, ByteBufferUtils.readVLong(b)); - b.flip(); - assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b))); - } - } - - @Test - public void testConsistencyWithHadoopVLong() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - for (long l : testNumbers) { - baos.reset(); - ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); - ByteBufferUtils.writeVLong(b, l); - String bufStr = Bytes.toStringBinary(b.array(), b.arrayOffset(), b.position()); - WritableUtils.writeVLong(dos, l); - String baosStr = Bytes.toStringBinary(baos.toByteArray()); - assertEquals(baosStr, bufStr); - } - } - - /** - * Test copying to stream from buffer. - */ - @Test - public void testMoveBufferToStream() throws IOException { - final int arrayOffset = 7; - final int initialPosition = 10; - final int endPadding = 5; - byte[] arrayWrapper = new byte[arrayOffset + initialPosition + array.length + endPadding]; - System.arraycopy(array, 0, arrayWrapper, arrayOffset + initialPosition, array.length); - ByteBuffer buffer = - ByteBuffer.wrap(arrayWrapper, arrayOffset, initialPosition + array.length).slice(); - assertEquals(initialPosition + array.length, buffer.limit()); - assertEquals(0, buffer.position()); - buffer.position(initialPosition); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ByteBufferUtils.moveBufferToStream(bos, buffer, array.length); - assertArrayEquals(array, bos.toByteArray()); - assertEquals(initialPosition + array.length, buffer.position()); - } - - /** - * Test copying to stream from buffer with offset. - * @throws IOException On test failure. - */ - @Test - public void testCopyToStreamWithOffset() throws IOException { - ByteBuffer buffer = ByteBuffer.wrap(array); - - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - - ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2, array.length / 2); - - byte[] returnedArray = bos.toByteArray(); - for (int i = 0; i < array.length / 2; ++i) { - int pos = array.length / 2 + i; - assertEquals(returnedArray[i], array[pos]); - } - } - - /** - * Test copying data from stream. - * @throws IOException On test failure. - */ - @Test - public void testCopyFromStream() throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(array.length); - ByteArrayInputStream bis = new ByteArrayInputStream(array); - DataInputStream dis = new DataInputStream(bis); - - ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2); - ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length - array.length / 2); - for (int i = 0; i < array.length; ++i) { - assertEquals(array[i], buffer.get(i)); - } - } - - /** - * Test copying from buffer. - */ - @Test - public void testCopyFromBuffer() { - ByteBuffer srcBuffer = ByteBuffer.allocate(array.length); - ByteBuffer dstBuffer = ByteBuffer.allocate(array.length); - srcBuffer.put(array); - - ByteBufferUtils.copyFromBufferToBuffer(srcBuffer, dstBuffer, array.length / 2, - array.length / 4); - for (int i = 0; i < array.length / 4; ++i) { - assertEquals(srcBuffer.get(i + array.length / 2), dstBuffer.get(i)); - } - } - - /** - * Test 7-bit encoding of integers. - * @throws IOException On test failure. - */ - @Test - public void testCompressedInt() throws IOException { - testCompressedInt(0); - testCompressedInt(Integer.MAX_VALUE); - testCompressedInt(Integer.MIN_VALUE); - - for (int i = 0; i < 3; i++) { - testCompressedInt((128 << i) - 1); - } - - for (int i = 0; i < 3; i++) { - testCompressedInt((128 << i)); - } - } - - /** - * Test how much bytes we need to store integer. - */ - @Test - public void testIntFitsIn() { - assertEquals(1, ByteBufferUtils.intFitsIn(0)); - assertEquals(1, ByteBufferUtils.intFitsIn(1)); - assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8)); - assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16)); - assertEquals(4, ByteBufferUtils.intFitsIn(-1)); - assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE)); - assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE)); - } - - /** - * Test how much bytes we need to store long. - */ - @Test - public void testLongFitsIn() { - assertEquals(1, ByteBufferUtils.longFitsIn(0)); - assertEquals(1, ByteBufferUtils.longFitsIn(1)); - assertEquals(3, ByteBufferUtils.longFitsIn(1L << 16)); - assertEquals(5, ByteBufferUtils.longFitsIn(1L << 32)); - assertEquals(8, ByteBufferUtils.longFitsIn(-1)); - assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE)); - assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE)); - } - - /** - * Test if we are comparing equal bytes. - */ - @Test - public void testArePartEqual() { - byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 }; - ByteBuffer buffer = ByteBuffer.wrap(array); - assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4)); - assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2)); - assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3)); - } - - /** - * Test serializing int to bytes - */ - @Test - public void testPutInt() { - testPutInt(0); - testPutInt(Integer.MAX_VALUE); - - for (int i = 0; i < 3; i++) { - testPutInt((128 << i) - 1); - } - - for (int i = 0; i < 3; i++) { - testPutInt((128 << i)); - } - } - - // Utility methods invoked from test methods - - private void testCompressedInt(int value) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ByteBufferUtils.putCompressedInt(bos, value); - ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - int parsedValue = ByteBufferUtils.readCompressedInt(bis); - assertEquals(value, parsedValue); - } - - private void testPutInt(int value) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - ByteBufferUtils.putInt(baos, value); - } catch (IOException e) { - throw new RuntimeException("Bug in putIn()", e); - } - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - DataInputStream dis = new DataInputStream(bais); - try { - assertEquals(dis.readInt(), value); - } catch (IOException e) { - throw new RuntimeException("Bug in test!", e); - } - } - - @Test - public void testToBytes() { - ByteBuffer buffer = ByteBuffer.allocate(5); - buffer.put(new byte[] { 0, 1, 2, 3, 4 }); - assertEquals(5, buffer.position()); - assertEquals(5, buffer.limit()); - byte[] copy = ByteBufferUtils.toBytes(buffer, 2); - assertArrayEquals(new byte[] { 2, 3, 4 }, copy); - assertEquals(5, buffer.position()); - assertEquals(5, buffer.limit()); - } - - @Test - public void testToPrimitiveTypes() { - ByteBuffer buffer = ByteBuffer.allocate(15); - long l = 988L; - int i = 135; - short s = 7; - buffer.putLong(l); - buffer.putShort(s); - buffer.putInt(i); - assertEquals(l, ByteBufferUtils.toLong(buffer, 0)); - assertEquals(s, ByteBufferUtils.toShort(buffer, 8)); - assertEquals(i, ByteBufferUtils.toInt(buffer, 10)); - } - - @Test - public void testCopyFromArrayToBuffer() { - byte[] b = new byte[15]; - b[0] = -1; - long l = 988L; - int i = 135; - short s = 7; - Bytes.putLong(b, 1, l); - Bytes.putShort(b, 9, s); - Bytes.putInt(b, 11, i); - ByteBuffer buffer = ByteBuffer.allocate(14); - ByteBufferUtils.copyFromArrayToBuffer(buffer, b, 1, 14); - buffer.rewind(); - assertEquals(l, buffer.getLong()); - assertEquals(s, buffer.getShort()); - assertEquals(i, buffer.getInt()); - } - - private void testCopyFromSrcToDestWithThreads(Object input, Object output, List lengthes, - List offsets) throws InterruptedException { - assertTrue((input instanceof ByteBuffer) || (input instanceof byte[])); - assertTrue((output instanceof ByteBuffer) || (output instanceof byte[])); - assertEquals(lengthes.size(), offsets.size()); - - final int threads = lengthes.size(); - CountDownLatch latch = new CountDownLatch(1); - List exes = new ArrayList<>(threads); - int oldInputPos = (input instanceof ByteBuffer) ? ((ByteBuffer) input).position() : 0; - int oldOutputPos = (output instanceof ByteBuffer) ? ((ByteBuffer) output).position() : 0; - for (int i = 0; i != threads; ++i) { - int offset = offsets.get(i); - int length = lengthes.get(i); - exes.add(() -> { - try { - latch.await(); - if (input instanceof ByteBuffer && output instanceof byte[]) { - ByteBufferUtils.copyFromBufferToArray((byte[]) output, (ByteBuffer) input, offset, - offset, length); - } - if (input instanceof byte[] && output instanceof ByteBuffer) { - ByteBufferUtils.copyFromArrayToBuffer((ByteBuffer) output, offset, (byte[]) input, - offset, length); - } - if (input instanceof ByteBuffer && output instanceof ByteBuffer) { - ByteBufferUtils.copyFromBufferToBuffer((ByteBuffer) input, (ByteBuffer) output, offset, - offset, length); - } - } catch (InterruptedException ex) { - throw new RuntimeException(ex); - } - }); - } - ExecutorService service = Executors.newFixedThreadPool(threads); - exes.forEach(service::execute); - latch.countDown(); - service.shutdown(); - assertTrue(service.awaitTermination(5, TimeUnit.SECONDS)); - if (input instanceof ByteBuffer) { - assertEquals(oldInputPos, ((ByteBuffer) input).position()); - } - if (output instanceof ByteBuffer) { - assertEquals(oldOutputPos, ((ByteBuffer) output).position()); - } - String inputString = (input instanceof ByteBuffer) - ? Bytes.toString(Bytes.toBytes((ByteBuffer) input)) - : Bytes.toString((byte[]) input); - String outputString = (output instanceof ByteBuffer) - ? Bytes.toString(Bytes.toBytes((ByteBuffer) output)) - : Bytes.toString((byte[]) output); - assertEquals(inputString, outputString); - } - - @Test - public void testCopyFromSrcToDestWithThreads() throws InterruptedException { - List words = - Arrays.asList(Bytes.toBytes("with"), Bytes.toBytes("great"), Bytes.toBytes("power"), - Bytes.toBytes("comes"), Bytes.toBytes("great"), Bytes.toBytes("responsibility")); - List lengthes = words.stream().map(v -> v.length).collect(Collectors.toList()); - List offsets = new ArrayList<>(words.size()); - for (int i = 0; i != words.size(); ++i) { - offsets.add(words.subList(0, i).stream().mapToInt(v -> v.length).sum()); - } - - int totalSize = words.stream().mapToInt(v -> v.length).sum(); - byte[] fullContent = new byte[totalSize]; - int offset = 0; - for (byte[] w : words) { - offset = Bytes.putBytes(fullContent, offset, w, 0, w.length); - } - - // test copyFromBufferToArray - for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - words.forEach(input::put); - byte[] output = new byte[totalSize]; - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - - // test copyFromArrayToBuffer - for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - byte[] input = fullContent; - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - - // test copyFromBufferToBuffer - for (ByteBuffer input : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - words.forEach(input::put); - for (ByteBuffer output : Arrays.asList(ByteBuffer.allocateDirect(totalSize), - ByteBuffer.allocate(totalSize))) { - testCopyFromSrcToDestWithThreads(input, output, lengthes, offsets); - } - } - } - - @Test - public void testCopyFromBufferToArray() { - ByteBuffer buffer = ByteBuffer.allocate(15); - buffer.put((byte) -1); - long l = 988L; - int i = 135; - short s = 7; - buffer.putShort(s); - buffer.putInt(i); - buffer.putLong(l); - byte[] b = new byte[15]; - ByteBufferUtils.copyFromBufferToArray(b, buffer, 1, 1, 14); - assertEquals(s, Bytes.toShort(b, 1)); - assertEquals(i, Bytes.toInt(b, 3)); - assertEquals(l, Bytes.toLong(b, 7)); - } - - @Test - public void testRelativeCopyFromBuffertoBuffer() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - fillBB(bb1, (byte) 5); - ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); - assertTrue(bb1.position() == bb2.position()); - assertTrue(bb1.limit() == bb2.limit()); - bb1 = ByteBuffer.allocateDirect(135); - bb2 = ByteBuffer.allocateDirect(135); - fillBB(bb1, (byte) 5); - ByteBufferUtils.copyFromBufferToBuffer(bb1, bb2); - assertTrue(bb1.position() == bb2.position()); - assertTrue(bb1.limit() == bb2.limit()); - } - - @Test - public void testCompareTo() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - byte[] b = new byte[71]; - fillBB(bb1, (byte) 5); - fillBB(bb2, (byte) 5); - fillArray(b, (byte) 5); - assertEquals(0, ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), b, 0, b.length) > 0); - bb2.put(134, (byte) 6); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) < 0); - bb2.put(6, (byte) 4); - assertTrue(ByteBufferUtils.compareTo(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining()) > 0); - // Assert reverse comparing BB and bytearray works. - ByteBuffer bb3 = ByteBuffer.allocate(135); - fillBB(bb3, (byte) 0); - byte[] b3 = new byte[135]; - fillArray(b3, (byte) 1); - int result = ByteBufferUtils.compareTo(b3, 0, b3.length, bb3, 0, bb3.remaining()); - assertTrue(result > 0); - result = ByteBufferUtils.compareTo(bb3, 0, bb3.remaining(), b3, 0, b3.length); - assertTrue(result < 0); - byte[] b4 = Bytes.toBytes("123"); - ByteBuffer bb4 = ByteBuffer.allocate(10 + b4.length); - for (int i = 10; i < bb4.capacity(); ++i) { - bb4.put(i, b4[i - 10]); - } - result = ByteBufferUtils.compareTo(b4, 0, b4.length, bb4, 10, b4.length); - assertEquals(0, result); - } - - @Test - public void testEquals() { - byte[] a = Bytes.toBytes("http://A"); - ByteBuffer bb = ByteBuffer.wrap(a); - - assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, - HConstants.EMPTY_BYTE_BUFFER, 0, 0)); - - assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, bb, 0, a.length)); - - assertFalse(ByteBufferUtils.equals(bb, 0, 0, HConstants.EMPTY_BYTE_BUFFER, 0, a.length)); - - assertTrue(ByteBufferUtils.equals(bb, 0, a.length, bb, 0, a.length)); - - assertTrue(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, - HConstants.EMPTY_BYTE_ARRAY, 0, 0)); - - assertFalse(ByteBufferUtils.equals(HConstants.EMPTY_BYTE_BUFFER, 0, 0, a, 0, a.length)); - - assertFalse(ByteBufferUtils.equals(bb, 0, a.length, HConstants.EMPTY_BYTE_ARRAY, 0, 0)); - - assertTrue(ByteBufferUtils.equals(bb, 0, a.length, a, 0, a.length)); - } - - @Test - public void testFindCommonPrefix() { - ByteBuffer bb1 = ByteBuffer.allocate(135); - ByteBuffer bb2 = ByteBuffer.allocate(135); - ByteBuffer bb3 = ByteBuffer.allocateDirect(135); - byte[] b = new byte[71]; - - fillBB(bb1, (byte) 5); - fillBB(bb2, (byte) 5); - fillBB(bb3, (byte) 5); - fillArray(b, (byte) 5); - - assertEquals(135, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - assertEquals(71, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); - assertEquals(135, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb3, 0, bb3.remaining())); - assertEquals(71, ByteBufferUtils.findCommonPrefix(bb3, 0, bb3.remaining(), b, 0, b.length)); - - b[13] = 9; - assertEquals(13, ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), b, 0, b.length)); - - bb2.put(134, (byte) 6); - assertEquals(134, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - - bb2.put(6, (byte) 4); - assertEquals(6, - ByteBufferUtils.findCommonPrefix(bb1, 0, bb1.remaining(), bb2, 0, bb2.remaining())); - } - - private static void fillBB(ByteBuffer bb, byte b) { - for (int i = bb.position(); i < bb.limit(); i++) { - bb.put(i, b); - } - } +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; - private static void fillArray(byte[] bb, byte b) { - for (int i = 0; i < bb.length; i++) { - bb[i] = b; - } - } +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestByteBufferUtils extends ByteBufferUtilsTestBase { } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java new file mode 100644 index 000000000000..c02db2142c1c --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtilsWoUnsafe.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mockStatic; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.mockito.MockedStatic; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestByteBufferUtilsWoUnsafe extends ByteBufferUtilsTestBase { + + @BeforeAll + public static void disableUnsafe() { + try (MockedStatic mocked = mockStatic(HBasePlatformDependent.class)) { + mocked.when(HBasePlatformDependent::isUnsafeAvailable).thenReturn(false); + mocked.when(HBasePlatformDependent::unaligned).thenReturn(false); + assertFalse(ByteBufferUtils.UNSAFE_AVAIL); + assertFalse(ByteBufferUtils.UNSAFE_UNALIGNED); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java index b74348959982..0122e91d7ea9 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java @@ -17,615 +17,12 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; -import org.apache.hadoop.io.WritableUtils; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ MiscTests.class, MediumTests.class }) -public class TestBytes { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestBytes.class); - - private static void setUnsafe(boolean value) throws Exception { - Field field = Bytes.class.getDeclaredField("UNSAFE_UNALIGNED"); - field.setAccessible(true); - - Field modifiersField = ReflectionUtils.getModifiersField(); - modifiersField.setAccessible(true); - int oldModifiers = field.getModifiers(); - modifiersField.setInt(field, oldModifiers & ~Modifier.FINAL); - try { - field.set(null, value); - } finally { - modifiersField.setInt(field, oldModifiers); - } - assertEquals(Bytes.UNSAFE_UNALIGNED, value); - } - - @Test - public void testShort() throws Exception { - testShort(false); - } - - @Test - public void testShortUnsafe() throws Exception { - testShort(true); - } - - private static void testShort(boolean unsafe) throws Exception { - setUnsafe(unsafe); - try { - for (short n : Arrays.asList(Short.MIN_VALUE, (short) -100, (short) -1, (short) 0, (short) 1, - (short) 300, Short.MAX_VALUE)) { - byte[] bytes = Bytes.toBytes(n); - assertEquals(Bytes.toShort(bytes, 0, bytes.length), n); - } - } finally { - setUnsafe(HBasePlatformDependent.unaligned()); - } - } - - @Test - public void testNullHashCode() { - byte[] b = null; - Exception ee = null; - try { - Bytes.hashCode(b); - } catch (Exception e) { - ee = e; - } - assertNotNull(ee); - } - - @Test - public void testAdd() { - byte[] a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - byte[] b = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - byte[] c = { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }; - byte[] result1 = Bytes.add(a, b, c); - byte[] result2 = Bytes.add(new byte[][] { a, b, c }); - assertEquals(0, Bytes.compareTo(result1, result2)); - } - - @Test - public void testSplit() { - byte[] lowest = Bytes.toBytes("AAA"); - byte[] middle = Bytes.toBytes("CCC"); - byte[] highest = Bytes.toBytes("EEE"); - byte[][] parts = Bytes.split(lowest, highest, 1); - for (byte[] bytes : parts) { - System.out.println(Bytes.toString(bytes)); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - // Now divide into three parts. Change highest so split is even. - highest = Bytes.toBytes("DDD"); - parts = Bytes.split(lowest, highest, 2); - for (byte[] part : parts) { - System.out.println(Bytes.toString(part)); - } - assertEquals(4, parts.length); - // Assert that 3rd part is 'CCC'. - assertTrue(Bytes.equals(parts[2], middle)); - } - - @Test - public void testSplit2() { - // More split tests. - byte[] lowest = Bytes.toBytes("http://A"); - byte[] highest = Bytes.toBytes("http://z"); - byte[] middle = Bytes.toBytes("http://]"); - byte[][] parts = Bytes.split(lowest, highest, 1); - for (byte[] part : parts) { - System.out.println(Bytes.toString(part)); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - } - - @Test - public void testSplit3() { - // Test invalid split cases - byte[] low = { 1, 1, 1 }; - byte[] high = { 1, 1, 3 }; - - // If swapped, should throw IAE - try { - Bytes.split(high, low, 1); - fail("Should not be able to split if low > high"); - } catch (IllegalArgumentException iae) { - // Correct - } - - // Single split should work - byte[][] parts = Bytes.split(low, high, 1); - for (int i = 0; i < parts.length; i++) { - System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i])); - } - assertEquals("Returned split should have 3 parts but has " + parts.length, 3, parts.length); - - // If split more than once, use additional byte to split - parts = Bytes.split(low, high, 2); - assertNotNull("Split with an additional byte", parts); - assertEquals(parts.length, low.length + 1); - - // Split 0 times should throw IAE - try { - Bytes.split(low, high, 0); - fail("Should not be able to split 0 times"); - } catch (IllegalArgumentException iae) { - // Correct - } - } - - @Test - public void testToInt() { - int[] ints = { -1, 123, Integer.MIN_VALUE, Integer.MAX_VALUE }; - for (int anInt : ints) { - byte[] b = Bytes.toBytes(anInt); - assertEquals(anInt, Bytes.toInt(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(anInt, Bytes.toInt(b2, 1)); - assertEquals(anInt, Bytes.toInt(b2, 1, Bytes.SIZEOF_INT)); - } - } - - @Test - public void testToLong() { - long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; - for (long aLong : longs) { - byte[] b = Bytes.toBytes(aLong); - assertEquals(aLong, Bytes.toLong(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(aLong, Bytes.toLong(b2, 1)); - assertEquals(aLong, Bytes.toLong(b2, 1, Bytes.SIZEOF_LONG)); - } - } - - @Test - public void testToFloat() { - float[] floats = { -1f, 123.123f, Float.MAX_VALUE }; - for (float aFloat : floats) { - byte[] b = Bytes.toBytes(aFloat); - assertEquals(aFloat, Bytes.toFloat(b), 0.0f); - byte[] b2 = bytesWithOffset(b); - assertEquals(aFloat, Bytes.toFloat(b2, 1), 0.0f); - } - } - - @Test - public void testToDouble() { - double[] doubles = { Double.MIN_VALUE, Double.MAX_VALUE }; - for (double aDouble : doubles) { - byte[] b = Bytes.toBytes(aDouble); - assertEquals(aDouble, Bytes.toDouble(b), 0.0); - byte[] b2 = bytesWithOffset(b); - assertEquals(aDouble, Bytes.toDouble(b2, 1), 0.0); - } - } - - @Test - public void testToBigDecimal() { - BigDecimal[] decimals = - { new BigDecimal("-1"), new BigDecimal("123.123"), new BigDecimal("123123123123") }; - for (BigDecimal decimal : decimals) { - byte[] b = Bytes.toBytes(decimal); - assertEquals(decimal, Bytes.toBigDecimal(b)); - byte[] b2 = bytesWithOffset(b); - assertEquals(decimal, Bytes.toBigDecimal(b2, 1, b.length)); - } - } - - private byte[] bytesWithOffset(byte[] src) { - // add one byte in front to test offset - byte[] result = new byte[src.length + 1]; - result[0] = (byte) 0xAA; - System.arraycopy(src, 0, result, 1, src.length); - return result; - } - - @Test - public void testToBytesForByteBuffer() { - byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - ByteBuffer target = ByteBuffer.wrap(array); - target.position(2); - target.limit(7); - - byte[] actual = Bytes.toBytes(target); - byte[] expected = { 0, 1, 2, 3, 4, 5, 6 }; - assertArrayEquals(expected, actual); - assertEquals(2, target.position()); - assertEquals(7, target.limit()); - - ByteBuffer target2 = target.slice(); - assertEquals(0, target2.position()); - assertEquals(5, target2.limit()); - - byte[] actual2 = Bytes.toBytes(target2); - byte[] expected2 = { 2, 3, 4, 5, 6 }; - assertArrayEquals(expected2, actual2); - assertEquals(0, target2.position()); - assertEquals(5, target2.limit()); - } - - @Test - public void testGetBytesForByteBuffer() { - byte[] array = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - ByteBuffer target = ByteBuffer.wrap(array); - target.position(2); - target.limit(7); - - byte[] actual = Bytes.getBytes(target); - byte[] expected = { 2, 3, 4, 5, 6 }; - assertArrayEquals(expected, actual); - assertEquals(2, target.position()); - assertEquals(7, target.limit()); - } - - @Test - public void testReadAsVLong() throws Exception { - long[] longs = { -1L, 123L, Long.MIN_VALUE, Long.MAX_VALUE }; - for (long aLong : longs) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream output = new DataOutputStream(baos); - WritableUtils.writeVLong(output, aLong); - byte[] long_bytes_no_offset = baos.toByteArray(); - assertEquals(aLong, Bytes.readAsVLong(long_bytes_no_offset, 0)); - byte[] long_bytes_with_offset = bytesWithOffset(long_bytes_no_offset); - assertEquals(aLong, Bytes.readAsVLong(long_bytes_with_offset, 1)); - } - } - - @Test - public void testToStringBinaryForBytes() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - String actual = Bytes.toStringBinary(array); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - - String actual2 = Bytes.toStringBinary(array, 2, 3); - String expected2 = "azA"; - assertEquals(expected2, actual2); - } - - @Test - public void testToStringBinaryForArrayBasedByteBuffer() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - ByteBuffer target = ByteBuffer.wrap(array); - String actual = Bytes.toStringBinary(target); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - } - - @Test - public void testToStringBinaryForReadOnlyByteBuffer() { - byte[] array = { '0', '9', 'a', 'z', 'A', 'Z', '@', 1 }; - ByteBuffer target = ByteBuffer.wrap(array).asReadOnlyBuffer(); - String actual = Bytes.toStringBinary(target); - String expected = "09azAZ@\\x01"; - assertEquals(expected, actual); - } - - @Test - public void testBinarySearch() { - byte[][] arr = { { 1 }, { 3 }, { 5 }, { 7 }, { 9 }, { 11 }, { 13 }, { 15 }, }; - byte[] key1 = { 3, 1 }; - byte[] key2 = { 4, 9 }; - byte[] key2_2 = { 4 }; - byte[] key3 = { 5, 11 }; - byte[] key4 = { 0 }; - byte[] key5 = { 2 }; - - assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1)); - assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1)); - assertEquals(-(2 + 1), Arrays.binarySearch(arr, key2_2, Bytes.BYTES_COMPARATOR)); - assertEquals(-(2 + 1), Bytes.binarySearch(arr, key2, 0, 1)); - assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1)); - assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1)); - assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1)); - assertEquals(-1, Bytes.binarySearch(arr, key4, 0, 1)); - assertEquals(-2, Bytes.binarySearch(arr, key5, 0, 1)); - - // Search for values to the left and to the right of each item in the array. - for (int i = 0; i < arr.length; ++i) { - assertEquals(-(i + 1), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] - 1) }, 0, 1)); - assertEquals(-(i + 2), Bytes.binarySearch(arr, new byte[] { (byte) (arr[i][0] + 1) }, 0, 1)); - } - } - - @Test - public void testToStringBytesBinaryReversible() { - byte[] randomBytes = new byte[1000]; - for (int i = 0; i < 1000; i++) { - Bytes.random(randomBytes); - verifyReversibleForBytes(randomBytes); - } - // some specific cases - verifyReversibleForBytes(new byte[] {}); - verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D' }); - verifyReversibleForBytes(new byte[] { '\\', 'x', 'A', 'D', '\\' }); - } - - private void verifyReversibleForBytes(byte[] originalBytes) { - String convertedString = Bytes.toStringBinary(originalBytes); - byte[] convertedBytes = Bytes.toBytesBinary(convertedString); - if (Bytes.compareTo(originalBytes, convertedBytes) != 0) { - fail("Not reversible for\nbyte[]: " + Arrays.toString(originalBytes) + ",\nStringBinary: " - + convertedString); - } - } - - @Test - public void testStartsWith() { - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("h"))); - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes(""))); - assertTrue(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("hello"))); - assertFalse(Bytes.startsWith(Bytes.toBytes("hello"), Bytes.toBytes("helloworld"))); - assertFalse(Bytes.startsWith(Bytes.toBytes(""), Bytes.toBytes("hello"))); - } - - @Test - public void testIncrementBytes() { - assertTrue(checkTestIncrementBytes(10, 1)); - assertTrue(checkTestIncrementBytes(12, 123435445)); - assertTrue(checkTestIncrementBytes(124634654, 1)); - assertTrue(checkTestIncrementBytes(10005460, 5005645)); - assertTrue(checkTestIncrementBytes(1, -1)); - assertTrue(checkTestIncrementBytes(10, -1)); - assertTrue(checkTestIncrementBytes(10, -5)); - assertTrue(checkTestIncrementBytes(1005435000, -5)); - assertTrue(checkTestIncrementBytes(10, -43657655)); - assertTrue(checkTestIncrementBytes(-1, 1)); - assertTrue(checkTestIncrementBytes(-26, 5034520)); - assertTrue(checkTestIncrementBytes(-10657200, 5)); - assertTrue(checkTestIncrementBytes(-12343250, 45376475)); - assertTrue(checkTestIncrementBytes(-10, -5)); - assertTrue(checkTestIncrementBytes(-12343250, -5)); - assertTrue(checkTestIncrementBytes(-12, -34565445)); - assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); - } - - private static boolean checkTestIncrementBytes(long val, long amount) { - byte[] value = Bytes.toBytes(val); - byte[] testValue = { -1, -1, -1, -1, -1, -1, -1, -1 }; - if (value[0] > 0) { - testValue = new byte[Bytes.SIZEOF_LONG]; - } - System.arraycopy(value, 0, testValue, testValue.length - value.length, value.length); - - long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); - - return (Bytes.toLong(testValue) + amount) == incrementResult; - } - - @Test - public void testFixedSizeString() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - Bytes.writeStringFixedSize(dos, "Hello", 5); - Bytes.writeStringFixedSize(dos, "World", 18); - Bytes.writeStringFixedSize(dos, "", 9); - - try { - // Use a long dash which is three bytes in UTF-8. If encoding happens - // using ISO-8859-1, this will fail. - Bytes.writeStringFixedSize(dos, "Too\u2013Long", 9); - fail("Exception expected"); - } catch (IOException ex) { - assertEquals( - "Trying to write 10 bytes (Too\\xE2\\x80\\x93Long) into a field of " + "length 9", - ex.getMessage()); - } - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - DataInputStream dis = new DataInputStream(bais); - assertEquals("Hello", Bytes.readStringFixedSize(dis, 5)); - assertEquals("World", Bytes.readStringFixedSize(dis, 18)); - assertEquals("", Bytes.readStringFixedSize(dis, 9)); - } - - @Test - public void testCopy() { - byte[] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - byte[] copy = Bytes.copy(bytes); - assertNotSame(bytes, copy); - assertTrue(Bytes.equals(bytes, copy)); - } - - @Test - public void testToBytesBinaryTrailingBackslashes() { - try { - Bytes.toBytesBinary("abc\\x00\\x01\\"); - } catch (StringIndexOutOfBoundsException ex) { - fail("Illegal string access: " + ex.getMessage()); - } - } - - @Test - public void testToStringBinary_toBytesBinary_Reversable() { - String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17)); - assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0); - } - - @Test - public void testUnsignedBinarySearch() { - byte[] bytes = new byte[] { 0, 5, 123, 127, -128, -100, -1 }; - Assert.assertEquals(1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 5)); - Assert.assertEquals(3, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 127)); - Assert.assertEquals(4, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -128)); - Assert.assertEquals(5, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -100)); - Assert.assertEquals(6, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -1)); - Assert.assertEquals(-1 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) 2)); - Assert.assertEquals(-6 - 1, Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte) -5)); - } - - @Test - public void testUnsignedIncrement() { - byte[] a = Bytes.toBytes(0); - int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0); - Assert.assertEquals(1, a2); - - byte[] b = Bytes.toBytes(-1); - byte[] actual = Bytes.unsignedCopyAndIncrement(b); - Assert.assertNotSame(b, actual); - byte[] expected = new byte[] { 1, 0, 0, 0, 0 }; - assertArrayEquals(expected, actual); - - byte[] c = Bytes.toBytes(255);// should wrap to the next significant byte - int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0); - Assert.assertEquals(256, c2); - } - - @Test - public void testIndexOf() { - byte[] array = Bytes.toBytes("hello"); - assertEquals(1, Bytes.indexOf(array, (byte) 'e')); - assertEquals(4, Bytes.indexOf(array, (byte) 'o')); - assertEquals(-1, Bytes.indexOf(array, (byte) 'a')); - assertEquals(0, Bytes.indexOf(array, Bytes.toBytes("hel"))); - assertEquals(2, Bytes.indexOf(array, Bytes.toBytes("ll"))); - assertEquals(-1, Bytes.indexOf(array, Bytes.toBytes("hll"))); - } - - @Test - public void testContains() { - byte[] array = Bytes.toBytes("hello world"); - assertTrue(Bytes.contains(array, (byte) 'e')); - assertTrue(Bytes.contains(array, (byte) 'd')); - assertFalse(Bytes.contains(array, (byte) 'a')); - assertTrue(Bytes.contains(array, Bytes.toBytes("world"))); - assertTrue(Bytes.contains(array, Bytes.toBytes("ello"))); - assertFalse(Bytes.contains(array, Bytes.toBytes("owo"))); - } - - @Test - public void testZero() { - byte[] array = Bytes.toBytes("hello"); - Bytes.zero(array); - for (byte b : array) { - assertEquals(0, b); - } - array = Bytes.toBytes("hello world"); - Bytes.zero(array, 2, 7); - assertFalse(array[0] == 0); - assertFalse(array[1] == 0); - for (int i = 2; i < 9; i++) { - assertEquals(0, array[i]); - } - for (int i = 9; i < array.length; i++) { - assertFalse(array[i] == 0); - } - } - - @Test - public void testPutBuffer() { - byte[] b = new byte[100]; - for (byte i = 0; i < 100; i++) { - Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[] { i })); - } - for (byte i = 0; i < 100; i++) { - Assert.assertEquals(i, b[i]); - } - } - - @Test - public void testToFromHex() { - List testStrings = new ArrayList<>(8); - testStrings.addAll(Arrays.asList("", "00", "A0", "ff", "FFffFFFFFFFFFF", "12", - "0123456789abcdef", "283462839463924623984692834692346ABCDFEDDCA0")); - for (String testString : testStrings) { - byte[] byteData = Bytes.fromHex(testString); - Assert.assertEquals(testString.length() / 2, byteData.length); - String result = Bytes.toHex(byteData); - Assert.assertTrue(testString.equalsIgnoreCase(result)); - } - - List testByteData = new ArrayList<>(5); - testByteData.addAll(Arrays.asList(new byte[0], new byte[1], new byte[10], - new byte[] { 1, 2, 3, 4, 5 }, new byte[] { (byte) 0xFF })); - Random rand = ThreadLocalRandom.current(); - for (int i = 0; i < 20; i++) { - byte[] bytes = new byte[rand.nextInt(100)]; - Bytes.random(bytes); - testByteData.add(bytes); - } - - for (byte[] testData : testByteData) { - String hexString = Bytes.toHex(testData); - Assert.assertEquals(testData.length * 2, hexString.length()); - byte[] result = Bytes.fromHex(hexString); - assertArrayEquals(testData, result); - } - } - - @Test - public void testFindCommonPrefix() throws Exception { - testFindCommonPrefix(false); - } - - @Test - public void testFindCommonPrefixUnsafe() throws Exception { - testFindCommonPrefix(true); - } - - private static void testFindCommonPrefix(boolean unsafe) throws Exception { - setUnsafe(unsafe); - try { - // tests for common prefixes less than 8 bytes in length (i.e. using non-vectorized path) - byte[] hello = Bytes.toBytes("hello"); - byte[] helloWorld = Bytes.toBytes("helloworld"); - - assertEquals(5, - Bytes.findCommonPrefix(hello, helloWorld, hello.length, helloWorld.length, 0, 0)); - assertEquals(5, Bytes.findCommonPrefix(hello, hello, hello.length, hello.length, 0, 0)); - assertEquals(3, - Bytes.findCommonPrefix(hello, hello, hello.length - 2, hello.length - 2, 2, 2)); - assertEquals(0, Bytes.findCommonPrefix(hello, hello, 0, 0, 0, 0)); - - // tests for common prefixes greater than 8 bytes in length which may use the vectorized path - byte[] hellohello = Bytes.toBytes("hellohello"); - byte[] hellohellohi = Bytes.toBytes("hellohellohi"); +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; - assertEquals(10, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, - hellohellohi.length, 0, 0)); - assertEquals(10, Bytes.findCommonPrefix(hellohellohi, hellohello, hellohellohi.length, - hellohello.length, 0, 0)); - assertEquals(10, - Bytes.findCommonPrefix(hellohello, hellohello, hellohello.length, hellohello.length, 0, 0)); +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBytes extends BytesTestBase { - hellohello[2] = 0; - assertEquals(2, Bytes.findCommonPrefix(hellohello, hellohellohi, hellohello.length, - hellohellohi.length, 0, 0)); - } finally { - setUnsafe(HBasePlatformDependent.unaligned()); - } - } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java new file mode 100644 index 000000000000..8aacab4b8514 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytesWoUnsafe.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mockStatic; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.mockito.MockedStatic; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestBytesWoUnsafe extends BytesTestBase { + + @BeforeAll + public static void disableUnsafe() { + try (MockedStatic mocked = mockStatic(HBasePlatformDependent.class)) { + mocked.when(HBasePlatformDependent::unaligned).thenReturn(false); + assertFalse(Bytes.UNSAFE_UNALIGNED); + } + } +} diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index ea332ba5d609..6585117cc5ff 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -317,6 +317,11 @@ mockito-core test + + org.mockito + mockito-inline + test + org.slf4j jcl-over-slf4j diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java index 2f0f15de6705..283981caa9bd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -21,13 +21,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.util.List; -import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -48,9 +44,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -412,38 +405,8 @@ public void testTestDir() throws Exception { assertTrue(hbt.cleanupTestDir()); } - @Test - public void testResolvePortConflict() throws Exception { - // raises port conflict between 1st call and 2nd call of randomPort() by mocking Random object - Random random = mock(Random.class); - when(random.nextInt(anyInt())).thenAnswer(new Answer() { - int[] numbers = { 1, 1, 2 }; - int count = 0; - - @Override - public Integer answer(InvocationOnMock invocation) { - int ret = numbers[count]; - count++; - return ret; - } - }); - - HBaseTestingUtility.PortAllocator.AvailablePortChecker portChecker = - mock(HBaseTestingUtility.PortAllocator.AvailablePortChecker.class); - when(portChecker.available(anyInt())).thenReturn(true); - - HBaseTestingUtility.PortAllocator portAllocator = - new HBaseTestingUtility.PortAllocator(random, portChecker); - - int port1 = portAllocator.randomFreePort(); - int port2 = portAllocator.randomFreePort(); - assertNotEquals(port1, port2); - Mockito.verify(random, Mockito.times(3)).nextInt(anyInt()); - } - @Test public void testOverridingOfDefaultPorts() throws Exception { - // confirm that default port properties being overridden to random Configuration defaultConfig = HBaseConfiguration.create(); defaultConfig.setInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPortAllocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPortAllocator.java new file mode 100644 index 000000000000..1fc62516844b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPortAllocator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Random; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +@Tag(MiscTests.TAG) +@Tag(SmallTests.TAG) +public class TestPortAllocator { + + @Test + public void testResolvePortConflict() throws Exception { + // raises port conflict between 1st call and 2nd call of randomPort() by mocking Random object + Random random = mock(Random.class); + when(random.nextInt(anyInt())).thenAnswer(new Answer() { + int[] numbers = { 1, 1, 2 }; + int count = 0; + + @Override + public Integer answer(InvocationOnMock invocation) { + int ret = numbers[count]; + count++; + return ret; + } + }); + + HBaseTestingUtility.PortAllocator.AvailablePortChecker portChecker = + mock(HBaseTestingUtility.PortAllocator.AvailablePortChecker.class); + when(portChecker.available(anyInt())).thenReturn(true); + + HBaseTestingUtility.PortAllocator portAllocator = + new HBaseTestingUtility.PortAllocator(random, portChecker); + + int port1 = portAllocator.randomFreePort(); + int port2 = portAllocator.randomFreePort(); + assertNotEquals(port1, port2); + verify(random, Mockito.times(3)).nextInt(anyInt()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/FromClientSide3TestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/FromClientSide3TestBase.java new file mode 100644 index 000000000000..cfaed98d6ba4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/FromClientSide3TestBase.java @@ -0,0 +1,1208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; + +public class FromClientSide3TestBase { + + private static final Logger LOG = LoggerFactory.getLogger(FromClientSide3TestBase.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static int WAITTABLE_MILLIS; + private static byte[] FAMILY; + private static int SLAVES; + private static byte[] ROW; + private static byte[] ANOTHERROW; + private static byte[] QUALIFIER; + private static byte[] VALUE; + private static byte[] COL_QUAL; + private static byte[] VAL_BYTES; + private static byte[] ROW_BYTES; + + private TableName tableName; + + protected static void startCluster() throws Exception { + WAITTABLE_MILLIS = 10000; + FAMILY = Bytes.toBytes("testFamily"); + SLAVES = 3; + ROW = Bytes.toBytes("testRow"); + ANOTHERROW = Bytes.toBytes("anotherrow"); + QUALIFIER = Bytes.toBytes("testQualifier"); + VALUE = Bytes.toBytes("testValue"); + COL_QUAL = Bytes.toBytes("f1"); + VAL_BYTES = Bytes.toBytes("v1"); + ROW_BYTES = Bytes.toBytes("r1"); + TEST_UTIL.startMiniCluster(SLAVES); + } + + @AfterAll + public static void shutdownCluster() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @BeforeEach + public void setUp(TestInfo testInfo) throws Exception { + tableName = TableName.valueOf(testInfo.getTestMethod().get().getName()); + } + + @AfterEach + public void tearDown() throws Exception { + for (TableDescriptor htd : TEST_UTIL.getAdmin().listTableDescriptors()) { + LOG.info("Tear down, remove table=" + htd.getTableName()); + TEST_UTIL.deleteTable(htd.getTableName()); + } + } + + private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) throws Exception { + Put put = new Put(row); + Random rand = ThreadLocalRandom.current(); + for (int i = 0; i < nPuts; i++) { + byte[] qualifier = Bytes.toBytes(rand.nextInt()); + byte[] value = Bytes.toBytes(rand.nextInt()); + put.addColumn(family, qualifier, value); + } + table.put(put); + } + + private void performMultiplePutAndFlush(HBaseAdmin admin, Table table, byte[] row, byte[] family, + int nFlushes, int nPuts) throws Exception { + + try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(table.getName())) { + // connection needed for poll-wait + HRegionLocation loc = locator.getRegionLocation(row, true); + AdminProtos.AdminService.BlockingInterface server = + ((ClusterConnection) admin.getConnection()).getAdmin(loc.getServerName()); + byte[] regName = loc.getRegionInfo().getRegionName(); + + for (int i = 0; i < nFlushes; i++) { + randomCFPuts(table, row, family, nPuts); + List sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY); + int sfCount = sf.size(); + + admin.flush(table.getName()); + } + } + } + + private static List toList(ResultScanner scanner) { + try { + List cells = new ArrayList<>(); + for (Result r : scanner) { + cells.addAll(r.listCells()); + } + return cells; + } finally { + scanner.close(); + } + } + + @Test + public void testScanAfterDeletingSpecifiedRow() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + byte[] row = Bytes.toBytes("SpecifiedRow"); + byte[] value0 = Bytes.toBytes("value_0"); + byte[] value1 = Bytes.toBytes("value_1"); + Put put = new Put(row); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + Delete d = new Delete(row); + table.delete(d); + put = new Put(row); + put.addColumn(FAMILY, null, value0); + table.put(put); + put = new Put(row); + put.addColumn(FAMILY, null, value1); + table.put(put); + List cells = toList(table.getScanner(new Scan())); + assertEquals(1, cells.size()); + assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + + cells = toList(table.getScanner(new Scan().addFamily(FAMILY))); + assertEquals(1, cells.size()); + assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + + cells = toList(table.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); + assertEquals(0, cells.size()); + + TEST_UTIL.getAdmin().flush(tableName); + cells = toList(table.getScanner(new Scan())); + assertEquals(1, cells.size()); + assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + + cells = toList(table.getScanner(new Scan().addFamily(FAMILY))); + assertEquals(1, cells.size()); + assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + + cells = toList(table.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); + assertEquals(0, cells.size()); + } + } + + @Test + public void testScanAfterDeletingSpecifiedRowV2() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + byte[] row = Bytes.toBytes("SpecifiedRow"); + byte[] qual0 = Bytes.toBytes("qual0"); + byte[] qual1 = Bytes.toBytes("qual1"); + long now = EnvironmentEdgeManager.currentTime(); + Delete d = new Delete(row, now); + table.delete(d); + + Put put = new Put(row); + put.addColumn(FAMILY, null, now + 1, VALUE); + table.put(put); + + put = new Put(row); + put.addColumn(FAMILY, qual1, now + 2, qual1); + table.put(put); + + put = new Put(row); + put.addColumn(FAMILY, qual0, now + 3, qual0); + table.put(put); + + Result r = table.get(new Get(row)); + assertEquals(3, r.size(), r.toString()); + assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); + assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); + + TEST_UTIL.getAdmin().flush(tableName); + r = table.get(new Get(row)); + assertEquals(3, r.size()); + assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); + assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); + } + } + + // override the config settings at the CF level and ensure priority + @Test + public void testAdvancedConfigOverride() throws Exception { + /* + * Overall idea: (1) create 3 store files and issue a compaction. config's compaction.min == 3, + * so should work. (2) Increase the compaction.min toggle in the HTD to 5 and modify table. If + * we use the HTD value instead of the default config value, adding 3 files and issuing a + * compaction SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 and modify + * table. The CF schema should override the Table schema and now cause a minor compaction. + */ + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); + + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + try (Admin admin = TEST_UTIL.getAdmin()) { + ClusterConnection connection = (ClusterConnection) TEST_UTIL.getConnection(); + + // Create 3 store files. + byte[] row = Bytes.toBytes(ThreadLocalRandom.current().nextInt()); + performMultiplePutAndFlush((HBaseAdmin) admin, table, row, FAMILY, 3, 100); + + try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { + // Verify we have multiple store files. + HRegionLocation loc = locator.getRegionLocation(row, true); + byte[] regionName = loc.getRegionInfo().getRegionName(); + AdminProtos.AdminService.BlockingInterface server = + connection.getAdmin(loc.getServerName()); + assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() > 1); + + // Issue a compaction request + admin.compact(tableName); + + // poll wait for the compactions to happen + for (int i = 0; i < 10 * 1000 / 40; ++i) { + // The number of store files after compaction should be lesser. + loc = locator.getRegionLocation(row, true); + if (!loc.getRegionInfo().isOffline()) { + regionName = loc.getRegionInfo().getRegionName(); + server = connection.getAdmin(loc.getServerName()); + if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1) { + break; + } + } + Thread.sleep(40); + } + // verify the compactions took place and that we didn't just time out + assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1); + + // change the compaction.min config option for this table to 5 + LOG.info("hbase.hstore.compaction.min should now be 5"); + HTableDescriptor htd = new HTableDescriptor(table.getTableDescriptor()); + htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); + admin.modifyTable(tableName, htd); + Pair st = admin.getAlterStatus(tableName); + while (null != st && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + st = admin.getAlterStatus(tableName); + } + LOG.info("alter status finished"); + + // Create 3 more store files. + performMultiplePutAndFlush((HBaseAdmin) admin, table, row, FAMILY, 3, 10); + + // Issue a compaction request + admin.compact(tableName); + + // This time, the compaction request should not happen + Thread.sleep(10 * 1000); + loc = locator.getRegionLocation(row, true); + regionName = loc.getRegionInfo().getRegionName(); + server = connection.getAdmin(loc.getServerName()); + int sfCount = ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size(); + assertTrue(sfCount > 1); + + // change an individual CF's config option to 2 & online schema update + LOG.info("hbase.hstore.compaction.min should now be 2"); + HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); + hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); + htd.modifyFamily(hcd); + admin.modifyTable(tableName, htd); + st = admin.getAlterStatus(tableName); + while (null != st && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + st = admin.getAlterStatus(tableName); + } + LOG.info("alter status finished"); + + // Issue a compaction request + admin.compact(tableName); + + // poll wait for the compactions to happen + for (int i = 0; i < 10 * 1000 / 40; ++i) { + loc = locator.getRegionLocation(row, true); + regionName = loc.getRegionInfo().getRegionName(); + try { + server = connection.getAdmin(loc.getServerName()); + if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount) { + break; + } + } catch (Exception e) { + LOG.debug("Waiting for region to come online: " + Bytes.toString(regionName)); + } + Thread.sleep(40); + } + + // verify the compaction took place and that we didn't just time out + assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount); + + // Finally, ensure that we can remove a custom config value after we made it + LOG.info("Removing CF config value"); + LOG.info("hbase.hstore.compaction.min should now be 5"); + hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); + hcd.setValue("hbase.hstore.compaction.min", null); + htd.modifyFamily(hcd); + admin.modifyTable(tableName, htd); + st = admin.getAlterStatus(tableName); + while (null != st && st.getFirst() > 0) { + LOG.debug(st.getFirst() + " regions left to update"); + Thread.sleep(40); + st = admin.getAlterStatus(tableName); + } + LOG.info("alter status finished"); + assertNull( + table.getTableDescriptor().getFamily(FAMILY).getValue("hbase.hstore.compaction.min")); + } + } + } + } + + @Test + public void testHTableBatchWithEmptyPut() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + List actions = (List) new ArrayList(); + Object[] results = new Object[2]; + // create an empty Put + Put put1 = new Put(ROW); + actions.add(put1); + + Put put2 = new Put(ANOTHERROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE); + actions.add(put2); + + table.batch(actions, results); + fail("Empty Put should have failed the batch call"); + } catch (IllegalArgumentException iae) { + } + } + + // Test Table.batch with large amount of mutations against the same key. + // It used to trigger read lock's "Maximum lock count exceeded" Error. + @Test + public void testHTableWithLargeBatch() throws IOException, InterruptedException { + int sixtyFourK = 64 * 1024; + List actions = new ArrayList(); + Object[] results = new Object[(sixtyFourK + 1) * 2]; + + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + for (int i = 0; i < sixtyFourK + 1; i++) { + Put put1 = new Put(ROW); + put1.addColumn(FAMILY, QUALIFIER, VALUE); + actions.add(put1); + + Put put2 = new Put(ANOTHERROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE); + actions.add(put2); + } + + table.batch(actions, results); + } + } + + @Test + public void testBatchWithRowMutation() throws Exception { + LOG.info("Starting testBatchWithRowMutation"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") }; + + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + RowMutations arm = RowMutations + .of(Collections.singletonList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[0], VALUE))); + Object[] batchResult = new Object[1]; + table.batch(Arrays.asList(arm), batchResult); + + Get g = new Get(ROW); + Result r = table.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); + + arm = RowMutations.of(Arrays.asList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[1], VALUE), + new Delete(ROW).addColumns(FAMILY, QUALIFIERS[0]))); + table.batch(Arrays.asList(arm), batchResult); + r = table.get(g); + assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); + assertNull(r.getValue(FAMILY, QUALIFIERS[0])); + + // Test that we get the correct remote exception for RowMutations from batch() + try { + arm = RowMutations.of(Collections.singletonList( + new Put(ROW).addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE))); + table.batch(Arrays.asList(arm), batchResult); + fail("Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException"); + } catch (RetriesExhaustedWithDetailsException e) { + String msg = e.getMessage(); + assertTrue(msg.contains("NoSuchColumnFamilyException")); + } + } + } + + @Test + public void testBatchWithCheckAndMutate() throws Exception { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + byte[] row1 = Bytes.toBytes("row1"); + byte[] row2 = Bytes.toBytes("row2"); + byte[] row3 = Bytes.toBytes("row3"); + byte[] row4 = Bytes.toBytes("row4"); + byte[] row5 = Bytes.toBytes("row5"); + byte[] row6 = Bytes.toBytes("row6"); + byte[] row7 = Bytes.toBytes("row7"); + + table + .put(Arrays.asList(new Put(row1).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")), + new Put(row2).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")), + new Put(row3).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")), + new Put(row4).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")), + new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")), + new Put(row6).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L)), + new Put(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g")))); + + CheckAndMutate checkAndMutate1 = + CheckAndMutate.newBuilder(row1).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) + .build(new RowMutations(row1) + .add((Mutation) new Put(row1).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("g"))) + .add((Mutation) new Delete(row1).addColumns(FAMILY, Bytes.toBytes("A"))) + .add(new Increment(row1).addColumn(FAMILY, Bytes.toBytes("C"), 3L)) + .add(new Append(row1).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); + Get get = new Get(row2).addColumn(FAMILY, Bytes.toBytes("B")); + RowMutations mutations = new RowMutations(row3) + .add((Mutation) new Delete(row3).addColumns(FAMILY, Bytes.toBytes("C"))) + .add((Mutation) new Put(row3).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes("f"))) + .add(new Increment(row3).addColumn(FAMILY, Bytes.toBytes("A"), 5L)) + .add(new Append(row3).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); + CheckAndMutate checkAndMutate2 = + CheckAndMutate.newBuilder(row4).ifEquals(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("a")) + .build(new Put(row4).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("h"))); + Put put = new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("f")); + CheckAndMutate checkAndMutate3 = + CheckAndMutate.newBuilder(row6).ifEquals(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L)) + .build(new Increment(row6).addColumn(FAMILY, Bytes.toBytes("F"), 1)); + CheckAndMutate checkAndMutate4 = + CheckAndMutate.newBuilder(row7).ifEquals(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g")) + .build(new Append(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g"))); + + List actions = Arrays.asList(checkAndMutate1, get, mutations, checkAndMutate2, put, + checkAndMutate3, checkAndMutate4); + Object[] results = new Object[actions.size()]; + table.batch(actions, results); + + CheckAndMutateResult checkAndMutateResult = (CheckAndMutateResult) results[0]; + assertTrue(checkAndMutateResult.isSuccess()); + assertEquals(3L, + Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("C")))); + assertEquals("d", + Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("D")))); + + assertEquals("b", Bytes.toString(((Result) results[1]).getValue(FAMILY, Bytes.toBytes("B")))); + + Result result = (Result) results[2]; + assertTrue(result.getExists()); + assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A")))); + assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); + + checkAndMutateResult = (CheckAndMutateResult) results[3]; + assertFalse(checkAndMutateResult.isSuccess()); + assertNull(checkAndMutateResult.getResult()); + + assertTrue(((Result) results[4]).isEmpty()); + + checkAndMutateResult = (CheckAndMutateResult) results[5]; + assertTrue(checkAndMutateResult.isSuccess()); + assertEquals(11, + Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("F")))); + + checkAndMutateResult = (CheckAndMutateResult) results[6]; + assertTrue(checkAndMutateResult.isSuccess()); + assertEquals("gg", + Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("G")))); + + result = table.get(new Get(row1)); + assertEquals("g", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); + assertNull(result.getValue(FAMILY, Bytes.toBytes("A"))); + assertEquals(3L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("C")))); + assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); + + result = table.get(new Get(row3)); + assertNull(result.getValue(FAMILY, Bytes.toBytes("C"))); + assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("F")))); + assertNull(Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C")))); + assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A")))); + assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); + + result = table.get(new Get(row4)); + assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); + + result = table.get(new Get(row5)); + assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); + + result = table.get(new Get(row6)); + assertEquals(11, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("F")))); + + result = table.get(new Get(row7)); + assertEquals("gg", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("G")))); + } + } + + @Test + public void testHTableExistsMethodSingleRegionSingleGet() + throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + // Test with a single region table. + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + + Get get = new Get(ROW); + + boolean exist = table.exists(get); + assertFalse(exist); + + table.put(put); + + exist = table.exists(get); + assertTrue(exist); + } + } + + @Test + public void testHTableExistsMethodSingleRegionMultipleGets() + throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + List gets = new ArrayList<>(); + gets.add(new Get(ROW)); + gets.add(new Get(ANOTHERROW)); + + boolean[] results = table.exists(gets); + assertTrue(results[0]); + assertFalse(results[1]); + } + } + + @Test + public void testHTableExistsBeforeGet() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + Get get = new Get(ROW); + + boolean exist = table.exists(get); + assertEquals(true, exist); + + Result result = table.get(get); + assertEquals(false, result.isEmpty()); + assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER))); + } + } + + @Test + public void testHTableExistsAllBeforeGet() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2")); + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + put = new Put(ROW2); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + Get get = new Get(ROW); + Get get2 = new Get(ROW2); + ArrayList getList = new ArrayList(2); + getList.add(get); + getList.add(get2); + + boolean[] exists = table.existsAll(getList); + assertEquals(true, exists[0]); + assertEquals(true, exists[1]); + + Result[] result = table.get(getList); + assertEquals(false, result[0].isEmpty()); + assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER))); + assertEquals(false, result[1].isEmpty()); + assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER))); + } + } + + @Test + public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 1, + new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255)) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + + Get get = new Get(ROW); + + boolean exist = table.exists(get); + assertFalse(exist); + + table.put(put); + + exist = table.exists(get); + assertTrue(exist); + } + } + + @Test + public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 1, + new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255)) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + List gets = new ArrayList<>(); + gets.add(new Get(ANOTHERROW)); + gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 }))); + gets.add(new Get(ROW)); + gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 }))); + + LOG.info("Calling exists"); + boolean[] results = table.existsAll(gets); + assertFalse(results[0]); + assertFalse(results[1]); + assertTrue(results[2]); + assertFalse(results[3]); + + // Test with the first region. + put = new Put(new byte[] { 0x00 }); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + gets = new ArrayList<>(); + gets.add(new Get(new byte[] { 0x00 })); + gets.add(new Get(new byte[] { 0x00, 0x00 })); + results = table.existsAll(gets); + assertTrue(results[0]); + assertFalse(results[1]); + + // Test with the last region + put = new Put(new byte[] { (byte) 0xff, (byte) 0xff }); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + gets = new ArrayList<>(); + gets.add(new Get(new byte[] { (byte) 0xff })); + gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff })); + gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff })); + results = table.existsAll(gets); + assertFalse(results[0]); + assertTrue(results[1]); + assertFalse(results[2]); + } + } + + @Test + public void testGetEmptyRow() throws Exception { + // Create a table and put in 1 row + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW_BYTES); + put.addColumn(FAMILY, COL_QUAL, VAL_BYTES); + table.put(put); + + // Try getting the row with an empty row key + Result res = null; + try { + res = table.get(new Get(new byte[0])); + fail(); + } catch (IllegalArgumentException e) { + // Expected. + } + assertTrue(res == null); + res = table.get(new Get(Bytes.toBytes("r1-not-exist"))); + assertTrue(res.isEmpty() == true); + res = table.get(new Get(ROW_BYTES)); + assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES)); + } + } + + @Test + public void testConnectionDefaultUsesCodec() throws Exception { + ClusterConnection con = (ClusterConnection) TEST_UTIL.getConnection(); + assertTrue(con.hasCellBlockSupport()); + } + + @Test + public void testPutWithPreBatchMutate() throws Exception { + testPreBatchMutate(tableName, () -> { + try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + t.put(put); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + } + + @Test + public void testRowMutationsWithPreBatchMutate() throws Exception { + testPreBatchMutate(tableName, () -> { + try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { + RowMutations rm = new RowMutations(ROW, 1); + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + rm.add(put); + t.mutateRow(rm); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + } + + private void testPreBatchMutate(TableName tableName, Runnable rn) throws Exception { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addCoprocessor(WaitingForScanObserver.class.getName()); + desc.addFamily(new HColumnDescriptor(FAMILY)); + TEST_UTIL.getAdmin().createTable(desc); + // Don't use waitTableAvailable(), because the scanner will mess up the co-processor + + ExecutorService service = Executors.newFixedThreadPool(2); + service.execute(rn); + final List cells = new ArrayList<>(); + service.execute(() -> { + try { + // waiting for update. + TimeUnit.SECONDS.sleep(3); + try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { + Scan scan = new Scan(); + try (ResultScanner scanner = t.getScanner(scan)) { + for (Result r : scanner) { + cells.addAll(Arrays.asList(r.rawCells())); + } + } + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException(ex); + } + }); + service.shutdown(); + service.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + assertEquals(0, cells.size(), "The write is blocking by RegionObserver#postBatchMutate" + + ", so the data is invisible to reader"); + TEST_UTIL.deleteTable(tableName); + } + + @Test + public void testLockLeakWithDelta() throws Exception, Throwable { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addCoprocessor(WaitingForMultiMutationsObserver.class.getName()); + desc.setConfiguration("hbase.rowlock.wait.duration", String.valueOf(5000)); + desc.addFamily(new HColumnDescriptor(FAMILY)); + TEST_UTIL.getAdmin().createTable(desc); + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + // new a connection for lower retry number. + Configuration copy = new Configuration(TEST_UTIL.getConfiguration()); + copy.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); + try (Connection con = ConnectionFactory.createConnection(copy)) { + HRegion region = (HRegion) find(tableName); + region.setTimeoutForWriteLock(10); + ExecutorService putService = Executors.newSingleThreadExecutor(); + putService.execute(() -> { + try (Table table = con.getTable(tableName)) { + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIER, VALUE); + // the put will be blocked by WaitingForMultiMutationsObserver. + table.put(put); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + ExecutorService appendService = Executors.newSingleThreadExecutor(); + appendService.execute(() -> { + Append append = new Append(ROW); + append.addColumn(FAMILY, QUALIFIER, VALUE); + try (Table table = con.getTable(tableName)) { + table.append(append); + fail("The APPEND should fail because the target lock is blocked by previous put"); + } catch (Exception ex) { + } + }); + appendService.shutdown(); + appendService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + WaitingForMultiMutationsObserver observer = + find(tableName, WaitingForMultiMutationsObserver.class); + observer.latch.countDown(); + putService.shutdown(); + putService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + try (Table table = con.getTable(tableName)) { + Result r = table.get(new Get(ROW)); + assertFalse(r.isEmpty()); + assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), VALUE)); + } + } + HRegion region = (HRegion) find(tableName); + int readLockCount = region.getReadLockCount(); + LOG.info("readLockCount:" + readLockCount); + assertEquals(0, readLockCount); + } + + @Test + public void testMultiRowMutations() throws Exception, Throwable { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addCoprocessor(MultiRowMutationEndpoint.class.getName()); + desc.addCoprocessor(WaitingForMultiMutationsObserver.class.getName()); + desc.setConfiguration("hbase.rowlock.wait.duration", String.valueOf(5000)); + desc.addFamily(new HColumnDescriptor(FAMILY)); + TEST_UTIL.getAdmin().createTable(desc); + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + // new a connection for lower retry number. + Configuration copy = new Configuration(TEST_UTIL.getConfiguration()); + copy.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); + try (Connection con = ConnectionFactory.createConnection(copy)) { + byte[] row = Bytes.toBytes("ROW-0"); + byte[] rowLocked = Bytes.toBytes("ROW-1"); + byte[] value0 = Bytes.toBytes("VALUE-0"); + byte[] value1 = Bytes.toBytes("VALUE-1"); + byte[] value2 = Bytes.toBytes("VALUE-2"); + assertNoLocks(tableName); + ExecutorService putService = Executors.newSingleThreadExecutor(); + putService.execute(() -> { + try (Table table = con.getTable(tableName)) { + Put put0 = new Put(rowLocked); + put0.addColumn(FAMILY, QUALIFIER, value0); + // the put will be blocked by WaitingForMultiMutationsObserver. + table.put(put0); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); + ExecutorService cpService = Executors.newSingleThreadExecutor(); + cpService.execute(() -> { + boolean threw; + Put put1 = new Put(row); + Put put2 = new Put(rowLocked); + put1.addColumn(FAMILY, QUALIFIER, value1); + put2.addColumn(FAMILY, QUALIFIER, value2); + try (Table table = con.getTable(tableName)) { + MultiRowMutationProtos.MutateRowsRequest request = + MultiRowMutationProtos.MutateRowsRequest.newBuilder() + .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( + org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, + put1)) + .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( + org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, + put2)) + .build(); + table.coprocessorService(MultiRowMutationProtos.MultiRowMutationService.class, ROW, ROW, + (MultiRowMutationProtos.MultiRowMutationService exe) -> { + ServerRpcController controller = new ServerRpcController(); + CoprocessorRpcUtils.BlockingRpcCallback< + MultiRowMutationProtos.MutateRowsResponse> rpcCallback = + new CoprocessorRpcUtils.BlockingRpcCallback<>(); + exe.mutateRows(controller, request, rpcCallback); + return rpcCallback.get(); + }); + threw = false; + } catch (Throwable ex) { + threw = true; + } + if (!threw) { + // Can't call fail() earlier because the catch would eat it. + fail("This cp should fail because the target lock is blocked by previous put"); + } + }); + cpService.shutdown(); + cpService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + WaitingForMultiMutationsObserver observer = + find(tableName, WaitingForMultiMutationsObserver.class); + observer.latch.countDown(); + putService.shutdown(); + putService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + try (Table table = con.getTable(tableName)) { + Get g0 = new Get(row); + Get g1 = new Get(rowLocked); + Result r0 = table.get(g0); + Result r1 = table.get(g1); + assertTrue(r0.isEmpty()); + assertFalse(r1.isEmpty()); + assertTrue(Bytes.equals(r1.getValue(FAMILY, QUALIFIER), value0)); + } + assertNoLocks(tableName); + } + } + + /** + * A test case for issue HBASE-17482 After combile seqid with mvcc readpoint, seqid/mvcc is + * acquired and stamped onto cells in the append thread, a countdown latch is used to ensure that + * happened before cells can be put into memstore. But the MVCCPreAssign patch(HBASE-16698) make + * the seqid/mvcc acquirement in handler thread and stamping in append thread No countdown latch + * to assure cells in memstore are stamped with seqid/mvcc. If cells without mvcc(A.K.A mvcc=0) + * are put into memstore, then a scanner with a smaller readpoint can see these data, which + * disobey the multi version concurrency control rules. This test case is to reproduce this + * scenario. + */ + @Test + public void testMVCCUsingMVCCPreAssign() throws IOException, InterruptedException { + try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + // put two row first to init the scanner + Put put = new Put(Bytes.toBytes("0")); + put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes("0")); + table.put(put); + put = new Put(Bytes.toBytes("00")); + put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes("0")); + table.put(put); + Scan scan = new Scan(); + scan.setTimeRange(0, Long.MAX_VALUE); + scan.setCaching(1); + ResultScanner scanner = table.getScanner(scan); + int rowNum = scanner.next() != null ? 1 : 0; + // the started scanner shouldn't see the rows put below + for (int i = 1; i < 1000; i++) { + put = new Put(Bytes.toBytes(String.valueOf(i))); + put.setDurability(Durability.ASYNC_WAL); + put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes(i)); + table.put(put); + } + for (Result result : scanner) { + rowNum++; + } + // scanner should only see two rows + assertEquals(2, rowNum); + scanner = table.getScanner(scan); + rowNum = 0; + for (Result result : scanner) { + rowNum++; + } + // the new scanner should see all rows + assertEquals(1001, rowNum); + } + } + + @Test + public void testPutThenGetWithMultipleThreads() throws Exception { + final int THREAD_NUM = 20; + final int ROUND_NUM = 10; + for (int round = 0; round < ROUND_NUM; round++) { + ArrayList threads = new ArrayList<>(THREAD_NUM); + final AtomicInteger successCnt = new AtomicInteger(0); + try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + for (int i = 0; i < THREAD_NUM; i++) { + final int index = i; + Thread t = new Thread(new Runnable() { + + @Override + public void run() { + final byte[] row = Bytes.toBytes("row-" + index); + final byte[] value = Bytes.toBytes("v" + index); + try { + Put put = new Put(row); + put.addColumn(FAMILY, QUALIFIER, value); + ht.put(put); + Get get = new Get(row); + Result result = ht.get(get); + byte[] returnedValue = result.getValue(FAMILY, QUALIFIER); + if (Bytes.equals(value, returnedValue)) { + successCnt.getAndIncrement(); + } else { + LOG.error("Should be equal but not, original value: " + Bytes.toString(value) + + ", returned value: " + + (returnedValue == null ? "null" : Bytes.toString(returnedValue))); + } + } catch (Throwable e) { + // do nothing + } + } + }); + threads.add(t); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + assertEquals(THREAD_NUM, successCnt.get(), "Not equal in round " + round); + } + TEST_UTIL.deleteTable(tableName); + } + } + + private static void assertNoLocks(final TableName tableName) + throws IOException, InterruptedException { + HRegion region = (HRegion) find(tableName); + assertEquals(0, region.getLockedRows().size()); + } + + private static HRegion find(final TableName tableName) throws IOException, InterruptedException { + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName); + List regions = rs.getRegions(tableName); + assertEquals(1, regions.size()); + return regions.get(0); + } + + private static T find(final TableName tableName, Class clz) + throws IOException, InterruptedException { + HRegion region = find(tableName); + Coprocessor cp = region.getCoprocessorHost().findCoprocessor(clz.getName()); + assertTrue(clz.isInstance(cp), "The cp instance should be " + clz.getName() + + ", current instance is " + cp.getClass().getName()); + return clz.cast(cp); + } + + public static class WaitingForMultiMutationsObserver + implements RegionCoprocessor, RegionObserver { + final CountDownLatch latch = new CountDownLatch(1); + + @Override + public Optional getRegionObserver() { + return Optional.of(this); + } + + @Override + public void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress miniBatchOp) throws IOException { + try { + latch.await(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } + } + } + + public static class WaitingForScanObserver implements RegionCoprocessor, RegionObserver { + private final CountDownLatch latch = new CountDownLatch(1); + + @Override + public Optional getRegionObserver() { + return Optional.of(this); + } + + @Override + public void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress miniBatchOp) throws IOException { + try { + // waiting for scanner + latch.await(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } + } + + @Override + public RegionScanner postScannerOpen(final ObserverContext e, + final Scan scan, final RegionScanner s) throws IOException { + latch.countDown(); + return s; + } + } + + static byte[] generateHugeValue(int size) { + Random rand = ThreadLocalRandom.current(); + byte[] value = new byte[size]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) rand.nextInt(256); + } + return value; + } + + @Test + public void testScanWithBatchSizeReturnIncompleteCells() + throws IOException, InterruptedException { + TableDescriptor hd = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setMaxVersions(3).build()) + .build(); + try (Table table = TEST_UTIL.createTable(hd, null)) { + TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); + + Put put = new Put(ROW); + put.addColumn(FAMILY, Bytes.toBytes(0), generateHugeValue(3 * 1024 * 1024)); + table.put(put); + + put = new Put(ROW); + put.addColumn(FAMILY, Bytes.toBytes(1), generateHugeValue(4 * 1024 * 1024)); + table.put(put); + + for (int i = 2; i < 5; i++) { + for (int version = 0; version < 2; version++) { + put = new Put(ROW); + put.addColumn(FAMILY, Bytes.toBytes(i), generateHugeValue(1024)); + table.put(put); + } + } + + Scan scan = new Scan(); + scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(3) + .setMaxResultSize(4 * 1024 * 1024); + Result result; + try (ResultScanner scanner = table.getScanner(scan)) { + List list = new ArrayList<>(); + /* + * The first scan rpc should return a result with 2 cells, because 3MB + 4MB > 4MB; The + * second scan rpc should return a result with 3 cells, because reach the batch limit = 3; + * The mayHaveMoreCellsInRow in last result should be false in the scan rpc. BTW, the + * moreResultsInRegion also would be false. Finally, the client should collect all the cells + * into two result: 2+3 -> 3+2; + */ + while ((result = scanner.next()) != null) { + list.add(result); + } + + assertEquals(5, list.stream().mapToInt(Result::size).sum()); + assertEquals(2, list.size()); + assertEquals(3, list.get(0).size()); + assertEquals(2, list.get(1).size()); + } + + scan = new Scan(); + scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(2) + .setMaxResultSize(4 * 1024 * 1024); + try (ResultScanner scanner = table.getScanner(scan)) { + List list = new ArrayList<>(); + while ((result = scanner.next()) != null) { + list.add(result); + } + assertEquals(5, list.stream().mapToInt(Result::size).sum()); + assertEquals(3, list.size()); + assertEquals(2, list.get(0).size()); + assertEquals(2, list.get(1).size()); + assertEquals(1, list.get(2).size()); + } + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index aed5ba707c0a..daad7ce31886 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -17,1209 +17,17 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.Coprocessor; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.RegionObserver; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -import org.apache.hadoop.hbase.ipc.ServerRpcController; -import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; -import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; - -@Category({ LargeTests.class, ClientTests.class }) -public class TestFromClientSide3 { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFromClientSide3.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide3.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final int WAITTABLE_MILLIS = 10000; - private static byte[] FAMILY = Bytes.toBytes("testFamily"); - private static int SLAVES = 3; - private static final byte[] ROW = Bytes.toBytes("testRow"); - private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow"); - private static final byte[] QUALIFIER = Bytes.toBytes("testQualifier"); - private static final byte[] VALUE = Bytes.toBytes("testValue"); - private static final byte[] COL_QUAL = Bytes.toBytes("f1"); - private static final byte[] VAL_BYTES = Bytes.toBytes("v1"); - private static final byte[] ROW_BYTES = Bytes.toBytes("r1"); - - @Rule - public TestName name = new TestName(); - private TableName tableName; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.startMiniCluster(SLAVES); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - tableName = TableName.valueOf(name.getMethodName()); - } - - /** - * @throws java.lang.Exception - */ - @After - public void tearDown() throws Exception { - for (HTableDescriptor htd : TEST_UTIL.getAdmin().listTables()) { - LOG.info("Tear down, remove table=" + htd.getTableName()); - TEST_UTIL.deleteTable(htd.getTableName()); - } - } - - private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) throws Exception { - Put put = new Put(row); - Random rand = ThreadLocalRandom.current(); - for (int i = 0; i < nPuts; i++) { - byte[] qualifier = Bytes.toBytes(rand.nextInt()); - byte[] value = Bytes.toBytes(rand.nextInt()); - put.addColumn(family, qualifier, value); - } - table.put(put); - } - - private void performMultiplePutAndFlush(HBaseAdmin admin, Table table, byte[] row, byte[] family, - int nFlushes, int nPuts) throws Exception { - - try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(table.getName())) { - // connection needed for poll-wait - HRegionLocation loc = locator.getRegionLocation(row, true); - AdminProtos.AdminService.BlockingInterface server = - ((ClusterConnection) admin.getConnection()).getAdmin(loc.getServerName()); - byte[] regName = loc.getRegionInfo().getRegionName(); - - for (int i = 0; i < nFlushes; i++) { - randomCFPuts(table, row, family, nPuts); - List sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY); - int sfCount = sf.size(); - - admin.flush(table.getName()); - } - } - } - - private static List toList(ResultScanner scanner) { - try { - List cells = new ArrayList<>(); - for (Result r : scanner) { - cells.addAll(r.listCells()); - } - return cells; - } finally { - scanner.close(); - } - } - - @Test - public void testScanAfterDeletingSpecifiedRow() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - byte[] row = Bytes.toBytes("SpecifiedRow"); - byte[] value0 = Bytes.toBytes("value_0"); - byte[] value1 = Bytes.toBytes("value_1"); - Put put = new Put(row); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - Delete d = new Delete(row); - table.delete(d); - put = new Put(row); - put.addColumn(FAMILY, null, value0); - table.put(put); - put = new Put(row); - put.addColumn(FAMILY, null, value1); - table.put(put); - List cells = toList(table.getScanner(new Scan())); - assertEquals(1, cells.size()); - assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - - cells = toList(table.getScanner(new Scan().addFamily(FAMILY))); - assertEquals(1, cells.size()); - assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - - cells = toList(table.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); - assertEquals(0, cells.size()); - - TEST_UTIL.getAdmin().flush(tableName); - cells = toList(table.getScanner(new Scan())); - assertEquals(1, cells.size()); - assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - - cells = toList(table.getScanner(new Scan().addFamily(FAMILY))); - assertEquals(1, cells.size()); - assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - - cells = toList(table.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); - assertEquals(0, cells.size()); - } - } - - @Test - public void testScanAfterDeletingSpecifiedRowV2() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - byte[] row = Bytes.toBytes("SpecifiedRow"); - byte[] qual0 = Bytes.toBytes("qual0"); - byte[] qual1 = Bytes.toBytes("qual1"); - long now = EnvironmentEdgeManager.currentTime(); - Delete d = new Delete(row, now); - table.delete(d); - - Put put = new Put(row); - put.addColumn(FAMILY, null, now + 1, VALUE); - table.put(put); - - put = new Put(row); - put.addColumn(FAMILY, qual1, now + 2, qual1); - table.put(put); - - put = new Put(row); - put.addColumn(FAMILY, qual0, now + 3, qual0); - table.put(put); - - Result r = table.get(new Get(row)); - assertEquals(r.toString(), 3, r.size()); - assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); - assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); - assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); - - TEST_UTIL.getAdmin().flush(tableName); - r = table.get(new Get(row)); - assertEquals(3, r.size()); - assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); - assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); - assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); - } - } - - // override the config settings at the CF level and ensure priority - @Test - public void testAdvancedConfigOverride() throws Exception { - /* - * Overall idea: (1) create 3 store files and issue a compaction. config's compaction.min == 3, - * so should work. (2) Increase the compaction.min toggle in the HTD to 5 and modify table. If - * we use the HTD value instead of the default config value, adding 3 files and issuing a - * compaction SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 and modify - * table. The CF schema should override the Table schema and now cause a minor compaction. - */ - TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); - - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - try (Admin admin = TEST_UTIL.getAdmin()) { - ClusterConnection connection = (ClusterConnection) TEST_UTIL.getConnection(); - - // Create 3 store files. - byte[] row = Bytes.toBytes(ThreadLocalRandom.current().nextInt()); - performMultiplePutAndFlush((HBaseAdmin) admin, table, row, FAMILY, 3, 100); - - try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { - // Verify we have multiple store files. - HRegionLocation loc = locator.getRegionLocation(row, true); - byte[] regionName = loc.getRegionInfo().getRegionName(); - AdminProtos.AdminService.BlockingInterface server = - connection.getAdmin(loc.getServerName()); - assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() > 1); - - // Issue a compaction request - admin.compact(tableName); - - // poll wait for the compactions to happen - for (int i = 0; i < 10 * 1000 / 40; ++i) { - // The number of store files after compaction should be lesser. - loc = locator.getRegionLocation(row, true); - if (!loc.getRegionInfo().isOffline()) { - regionName = loc.getRegionInfo().getRegionName(); - server = connection.getAdmin(loc.getServerName()); - if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1) { - break; - } - } - Thread.sleep(40); - } - // verify the compactions took place and that we didn't just time out - assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1); - - // change the compaction.min config option for this table to 5 - LOG.info("hbase.hstore.compaction.min should now be 5"); - HTableDescriptor htd = new HTableDescriptor(table.getTableDescriptor()); - htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); - admin.modifyTable(tableName, htd); - Pair st = admin.getAlterStatus(tableName); - while (null != st && st.getFirst() > 0) { - LOG.debug(st.getFirst() + " regions left to update"); - Thread.sleep(40); - st = admin.getAlterStatus(tableName); - } - LOG.info("alter status finished"); - - // Create 3 more store files. - performMultiplePutAndFlush((HBaseAdmin) admin, table, row, FAMILY, 3, 10); - - // Issue a compaction request - admin.compact(tableName); - - // This time, the compaction request should not happen - Thread.sleep(10 * 1000); - loc = locator.getRegionLocation(row, true); - regionName = loc.getRegionInfo().getRegionName(); - server = connection.getAdmin(loc.getServerName()); - int sfCount = ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size(); - assertTrue(sfCount > 1); - - // change an individual CF's config option to 2 & online schema update - LOG.info("hbase.hstore.compaction.min should now be 2"); - HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); - hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); - htd.modifyFamily(hcd); - admin.modifyTable(tableName, htd); - st = admin.getAlterStatus(tableName); - while (null != st && st.getFirst() > 0) { - LOG.debug(st.getFirst() + " regions left to update"); - Thread.sleep(40); - st = admin.getAlterStatus(tableName); - } - LOG.info("alter status finished"); - - // Issue a compaction request - admin.compact(tableName); - - // poll wait for the compactions to happen - for (int i = 0; i < 10 * 1000 / 40; ++i) { - loc = locator.getRegionLocation(row, true); - regionName = loc.getRegionInfo().getRegionName(); - try { - server = connection.getAdmin(loc.getServerName()); - if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount) { - break; - } - } catch (Exception e) { - LOG.debug("Waiting for region to come online: " + Bytes.toString(regionName)); - } - Thread.sleep(40); - } - - // verify the compaction took place and that we didn't just time out - assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount); - - // Finally, ensure that we can remove a custom config value after we made it - LOG.info("Removing CF config value"); - LOG.info("hbase.hstore.compaction.min should now be 5"); - hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); - hcd.setValue("hbase.hstore.compaction.min", null); - htd.modifyFamily(hcd); - admin.modifyTable(tableName, htd); - st = admin.getAlterStatus(tableName); - while (null != st && st.getFirst() > 0) { - LOG.debug(st.getFirst() + " regions left to update"); - Thread.sleep(40); - st = admin.getAlterStatus(tableName); - } - LOG.info("alter status finished"); - assertNull( - table.getTableDescriptor().getFamily(FAMILY).getValue("hbase.hstore.compaction.min")); - } - } - } - } - - @Test - public void testHTableBatchWithEmptyPut() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - List actions = (List) new ArrayList(); - Object[] results = new Object[2]; - // create an empty Put - Put put1 = new Put(ROW); - actions.add(put1); - - Put put2 = new Put(ANOTHERROW); - put2.addColumn(FAMILY, QUALIFIER, VALUE); - actions.add(put2); - - table.batch(actions, results); - fail("Empty Put should have failed the batch call"); - } catch (IllegalArgumentException iae) { - } - } - - // Test Table.batch with large amount of mutations against the same key. - // It used to trigger read lock's "Maximum lock count exceeded" Error. - @Test - public void testHTableWithLargeBatch() throws IOException, InterruptedException { - int sixtyFourK = 64 * 1024; - List actions = new ArrayList(); - Object[] results = new Object[(sixtyFourK + 1) * 2]; - - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - for (int i = 0; i < sixtyFourK + 1; i++) { - Put put1 = new Put(ROW); - put1.addColumn(FAMILY, QUALIFIER, VALUE); - actions.add(put1); - - Put put2 = new Put(ANOTHERROW); - put2.addColumn(FAMILY, QUALIFIER, VALUE); - actions.add(put2); - } - - table.batch(actions, results); - } - } - - @Test - public void testBatchWithRowMutation() throws Exception { - LOG.info("Starting testBatchWithRowMutation"); - byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") }; - - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - RowMutations arm = RowMutations - .of(Collections.singletonList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[0], VALUE))); - Object[] batchResult = new Object[1]; - table.batch(Arrays.asList(arm), batchResult); - - Get g = new Get(ROW); - Result r = table.get(g); - assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); - - arm = RowMutations.of(Arrays.asList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[1], VALUE), - new Delete(ROW).addColumns(FAMILY, QUALIFIERS[0]))); - table.batch(Arrays.asList(arm), batchResult); - r = table.get(g); - assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); - assertNull(r.getValue(FAMILY, QUALIFIERS[0])); - - // Test that we get the correct remote exception for RowMutations from batch() - try { - arm = RowMutations.of(Collections.singletonList( - new Put(ROW).addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE))); - table.batch(Arrays.asList(arm), batchResult); - fail("Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException"); - } catch (RetriesExhaustedWithDetailsException e) { - String msg = e.getMessage(); - assertTrue(msg.contains("NoSuchColumnFamilyException")); - } - } - } - - @Test - public void testBatchWithCheckAndMutate() throws Exception { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - byte[] row1 = Bytes.toBytes("row1"); - byte[] row2 = Bytes.toBytes("row2"); - byte[] row3 = Bytes.toBytes("row3"); - byte[] row4 = Bytes.toBytes("row4"); - byte[] row5 = Bytes.toBytes("row5"); - byte[] row6 = Bytes.toBytes("row6"); - byte[] row7 = Bytes.toBytes("row7"); - - table - .put(Arrays.asList(new Put(row1).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")), - new Put(row2).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")), - new Put(row3).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")), - new Put(row4).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")), - new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")), - new Put(row6).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L)), - new Put(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g")))); - - CheckAndMutate checkAndMutate1 = - CheckAndMutate.newBuilder(row1).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) - .build(new RowMutations(row1) - .add((Mutation) new Put(row1).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("g"))) - .add((Mutation) new Delete(row1).addColumns(FAMILY, Bytes.toBytes("A"))) - .add(new Increment(row1).addColumn(FAMILY, Bytes.toBytes("C"), 3L)) - .add(new Append(row1).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); - Get get = new Get(row2).addColumn(FAMILY, Bytes.toBytes("B")); - RowMutations mutations = new RowMutations(row3) - .add((Mutation) new Delete(row3).addColumns(FAMILY, Bytes.toBytes("C"))) - .add((Mutation) new Put(row3).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes("f"))) - .add(new Increment(row3).addColumn(FAMILY, Bytes.toBytes("A"), 5L)) - .add(new Append(row3).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); - CheckAndMutate checkAndMutate2 = - CheckAndMutate.newBuilder(row4).ifEquals(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("a")) - .build(new Put(row4).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("h"))); - Put put = new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("f")); - CheckAndMutate checkAndMutate3 = - CheckAndMutate.newBuilder(row6).ifEquals(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L)) - .build(new Increment(row6).addColumn(FAMILY, Bytes.toBytes("F"), 1)); - CheckAndMutate checkAndMutate4 = - CheckAndMutate.newBuilder(row7).ifEquals(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g")) - .build(new Append(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g"))); - - List actions = Arrays.asList(checkAndMutate1, get, mutations, checkAndMutate2, put, - checkAndMutate3, checkAndMutate4); - Object[] results = new Object[actions.size()]; - table.batch(actions, results); - - CheckAndMutateResult checkAndMutateResult = (CheckAndMutateResult) results[0]; - assertTrue(checkAndMutateResult.isSuccess()); - assertEquals(3L, - Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("C")))); - assertEquals("d", - Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("D")))); - - assertEquals("b", Bytes.toString(((Result) results[1]).getValue(FAMILY, Bytes.toBytes("B")))); - - Result result = (Result) results[2]; - assertTrue(result.getExists()); - assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A")))); - assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); - - checkAndMutateResult = (CheckAndMutateResult) results[3]; - assertFalse(checkAndMutateResult.isSuccess()); - assertNull(checkAndMutateResult.getResult()); - - assertTrue(((Result) results[4]).isEmpty()); - - checkAndMutateResult = (CheckAndMutateResult) results[5]; - assertTrue(checkAndMutateResult.isSuccess()); - assertEquals(11, - Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("F")))); - - checkAndMutateResult = (CheckAndMutateResult) results[6]; - assertTrue(checkAndMutateResult.isSuccess()); - assertEquals("gg", - Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("G")))); - - result = table.get(new Get(row1)); - assertEquals("g", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); - assertNull(result.getValue(FAMILY, Bytes.toBytes("A"))); - assertEquals(3L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("C")))); - assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); - - result = table.get(new Get(row3)); - assertNull(result.getValue(FAMILY, Bytes.toBytes("C"))); - assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("F")))); - assertNull(Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C")))); - assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A")))); - assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); - - result = table.get(new Get(row4)); - assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); - - result = table.get(new Get(row5)); - assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); - - result = table.get(new Get(row6)); - assertEquals(11, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("F")))); - - result = table.get(new Get(row7)); - assertEquals("gg", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("G")))); - } - } - - @Test - public void testHTableExistsMethodSingleRegionSingleGet() - throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - // Test with a single region table. - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - - Get get = new Get(ROW); - - boolean exist = table.exists(get); - assertFalse(exist); - - table.put(put); - - exist = table.exists(get); - assertTrue(exist); - } - } - - @Test - public void testHTableExistsMethodSingleRegionMultipleGets() - throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - List gets = new ArrayList<>(); - gets.add(new Get(ROW)); - gets.add(new Get(ANOTHERROW)); - - boolean[] results = table.exists(gets); - assertTrue(results[0]); - assertFalse(results[1]); - } - } - - @Test - public void testHTableExistsBeforeGet() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - Get get = new Get(ROW); - - boolean exist = table.exists(get); - assertEquals(true, exist); - - Result result = table.get(get); - assertEquals(false, result.isEmpty()); - assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER))); - } - } - - @Test - public void testHTableExistsAllBeforeGet() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2")); - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - put = new Put(ROW2); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - Get get = new Get(ROW); - Get get2 = new Get(ROW2); - ArrayList getList = new ArrayList(2); - getList.add(get); - getList.add(get2); - - boolean[] exists = table.existsAll(getList); - assertEquals(true, exists[0]); - assertEquals(true, exists[1]); - - Result[] result = table.get(getList); - assertEquals(false, result[0].isEmpty()); - assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER))); - assertEquals(false, result[1].isEmpty()); - assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER))); - } - } - - @Test - public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 1, - new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255)) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - - Get get = new Get(ROW); - - boolean exist = table.exists(get); - assertFalse(exist); - - table.put(put); - - exist = table.exists(get); - assertTrue(exist); - } - } - - @Test - public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 1, - new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255)) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - List gets = new ArrayList<>(); - gets.add(new Get(ANOTHERROW)); - gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 }))); - gets.add(new Get(ROW)); - gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 }))); - - LOG.info("Calling exists"); - boolean[] results = table.existsAll(gets); - assertFalse(results[0]); - assertFalse(results[1]); - assertTrue(results[2]); - assertFalse(results[3]); - - // Test with the first region. - put = new Put(new byte[] { 0x00 }); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - gets = new ArrayList<>(); - gets.add(new Get(new byte[] { 0x00 })); - gets.add(new Get(new byte[] { 0x00, 0x00 })); - results = table.existsAll(gets); - assertTrue(results[0]); - assertFalse(results[1]); - - // Test with the last region - put = new Put(new byte[] { (byte) 0xff, (byte) 0xff }); - put.addColumn(FAMILY, QUALIFIER, VALUE); - table.put(put); - - gets = new ArrayList<>(); - gets.add(new Get(new byte[] { (byte) 0xff })); - gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff })); - gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff })); - results = table.existsAll(gets); - assertFalse(results[0]); - assertTrue(results[1]); - assertFalse(results[2]); - } - } - - @Test - public void testGetEmptyRow() throws Exception { - // Create a table and put in 1 row - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW_BYTES); - put.addColumn(FAMILY, COL_QUAL, VAL_BYTES); - table.put(put); - - // Try getting the row with an empty row key - Result res = null; - try { - res = table.get(new Get(new byte[0])); - fail(); - } catch (IllegalArgumentException e) { - // Expected. - } - assertTrue(res == null); - res = table.get(new Get(Bytes.toBytes("r1-not-exist"))); - assertTrue(res.isEmpty() == true); - res = table.get(new Get(ROW_BYTES)); - assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES)); - } - } - - @Test - public void testConnectionDefaultUsesCodec() throws Exception { - ClusterConnection con = (ClusterConnection) TEST_UTIL.getConnection(); - assertTrue(con.hasCellBlockSupport()); - } - - @Test - public void testPutWithPreBatchMutate() throws Exception { - testPreBatchMutate(tableName, () -> { - try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - t.put(put); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - }); - } - - @Test - public void testRowMutationsWithPreBatchMutate() throws Exception { - testPreBatchMutate(tableName, () -> { - try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { - RowMutations rm = new RowMutations(ROW, 1); - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - rm.add(put); - t.mutateRow(rm); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - }); - } - - private void testPreBatchMutate(TableName tableName, Runnable rn) throws Exception { - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addCoprocessor(WaitingForScanObserver.class.getName()); - desc.addFamily(new HColumnDescriptor(FAMILY)); - TEST_UTIL.getAdmin().createTable(desc); - // Don't use waitTableAvailable(), because the scanner will mess up the co-processor - - ExecutorService service = Executors.newFixedThreadPool(2); - service.execute(rn); - final List cells = new ArrayList<>(); - service.execute(() -> { - try { - // waiting for update. - TimeUnit.SECONDS.sleep(3); - try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { - Scan scan = new Scan(); - try (ResultScanner scanner = t.getScanner(scan)) { - for (Result r : scanner) { - cells.addAll(Arrays.asList(r.rawCells())); - } - } - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException(ex); - } - }); - service.shutdown(); - service.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - assertEquals("The write is blocking by RegionObserver#postBatchMutate" - + ", so the data is invisible to reader", 0, cells.size()); - TEST_UTIL.deleteTable(tableName); - } - - @Test - public void testLockLeakWithDelta() throws Exception, Throwable { - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addCoprocessor(WaitingForMultiMutationsObserver.class.getName()); - desc.setConfiguration("hbase.rowlock.wait.duration", String.valueOf(5000)); - desc.addFamily(new HColumnDescriptor(FAMILY)); - TEST_UTIL.getAdmin().createTable(desc); - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - // new a connection for lower retry number. - Configuration copy = new Configuration(TEST_UTIL.getConfiguration()); - copy.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); - try (Connection con = ConnectionFactory.createConnection(copy)) { - HRegion region = (HRegion) find(tableName); - region.setTimeoutForWriteLock(10); - ExecutorService putService = Executors.newSingleThreadExecutor(); - putService.execute(() -> { - try (Table table = con.getTable(tableName)) { - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIER, VALUE); - // the put will be blocked by WaitingForMultiMutationsObserver. - table.put(put); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - }); - ExecutorService appendService = Executors.newSingleThreadExecutor(); - appendService.execute(() -> { - Append append = new Append(ROW); - append.addColumn(FAMILY, QUALIFIER, VALUE); - try (Table table = con.getTable(tableName)) { - table.append(append); - fail("The APPEND should fail because the target lock is blocked by previous put"); - } catch (Exception ex) { - } - }); - appendService.shutdown(); - appendService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - WaitingForMultiMutationsObserver observer = - find(tableName, WaitingForMultiMutationsObserver.class); - observer.latch.countDown(); - putService.shutdown(); - putService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - try (Table table = con.getTable(tableName)) { - Result r = table.get(new Get(ROW)); - assertFalse(r.isEmpty()); - assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), VALUE)); - } - } - HRegion region = (HRegion) find(tableName); - int readLockCount = region.getReadLockCount(); - LOG.info("readLockCount:" + readLockCount); - assertEquals(0, readLockCount); - } - - @Test - public void testMultiRowMutations() throws Exception, Throwable { - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addCoprocessor(MultiRowMutationEndpoint.class.getName()); - desc.addCoprocessor(WaitingForMultiMutationsObserver.class.getName()); - desc.setConfiguration("hbase.rowlock.wait.duration", String.valueOf(5000)); - desc.addFamily(new HColumnDescriptor(FAMILY)); - TEST_UTIL.getAdmin().createTable(desc); - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - // new a connection for lower retry number. - Configuration copy = new Configuration(TEST_UTIL.getConfiguration()); - copy.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); - try (Connection con = ConnectionFactory.createConnection(copy)) { - byte[] row = Bytes.toBytes("ROW-0"); - byte[] rowLocked = Bytes.toBytes("ROW-1"); - byte[] value0 = Bytes.toBytes("VALUE-0"); - byte[] value1 = Bytes.toBytes("VALUE-1"); - byte[] value2 = Bytes.toBytes("VALUE-2"); - assertNoLocks(tableName); - ExecutorService putService = Executors.newSingleThreadExecutor(); - putService.execute(() -> { - try (Table table = con.getTable(tableName)) { - Put put0 = new Put(rowLocked); - put0.addColumn(FAMILY, QUALIFIER, value0); - // the put will be blocked by WaitingForMultiMutationsObserver. - table.put(put0); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - }); - ExecutorService cpService = Executors.newSingleThreadExecutor(); - cpService.execute(() -> { - boolean threw; - Put put1 = new Put(row); - Put put2 = new Put(rowLocked); - put1.addColumn(FAMILY, QUALIFIER, value1); - put2.addColumn(FAMILY, QUALIFIER, value2); - try (Table table = con.getTable(tableName)) { - MultiRowMutationProtos.MutateRowsRequest request = - MultiRowMutationProtos.MutateRowsRequest.newBuilder() - .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( - org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, - put1)) - .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( - org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, - put2)) - .build(); - table.coprocessorService(MultiRowMutationProtos.MultiRowMutationService.class, ROW, ROW, - (MultiRowMutationProtos.MultiRowMutationService exe) -> { - ServerRpcController controller = new ServerRpcController(); - CoprocessorRpcUtils.BlockingRpcCallback< - MultiRowMutationProtos.MutateRowsResponse> rpcCallback = - new CoprocessorRpcUtils.BlockingRpcCallback<>(); - exe.mutateRows(controller, request, rpcCallback); - return rpcCallback.get(); - }); - threw = false; - } catch (Throwable ex) { - threw = true; - } - if (!threw) { - // Can't call fail() earlier because the catch would eat it. - fail("This cp should fail because the target lock is blocked by previous put"); - } - }); - cpService.shutdown(); - cpService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - WaitingForMultiMutationsObserver observer = - find(tableName, WaitingForMultiMutationsObserver.class); - observer.latch.countDown(); - putService.shutdown(); - putService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - try (Table table = con.getTable(tableName)) { - Get g0 = new Get(row); - Get g1 = new Get(rowLocked); - Result r0 = table.get(g0); - Result r1 = table.get(g1); - assertTrue(r0.isEmpty()); - assertFalse(r1.isEmpty()); - assertTrue(Bytes.equals(r1.getValue(FAMILY, QUALIFIER), value0)); - } - assertNoLocks(tableName); - } - } - - /** - * A test case for issue HBASE-17482 After combile seqid with mvcc readpoint, seqid/mvcc is - * acquired and stamped onto cells in the append thread, a countdown latch is used to ensure that - * happened before cells can be put into memstore. But the MVCCPreAssign patch(HBASE-16698) make - * the seqid/mvcc acquirement in handler thread and stamping in append thread No countdown latch - * to assure cells in memstore are stamped with seqid/mvcc. If cells without mvcc(A.K.A mvcc=0) - * are put into memstore, then a scanner with a smaller readpoint can see these data, which - * disobey the multi version concurrency control rules. This test case is to reproduce this - * scenario. - */ - @Test - public void testMVCCUsingMVCCPreAssign() throws IOException, InterruptedException { - try (Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY })) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - // put two row first to init the scanner - Put put = new Put(Bytes.toBytes("0")); - put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes("0")); - table.put(put); - put = new Put(Bytes.toBytes("00")); - put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes("0")); - table.put(put); - Scan scan = new Scan(); - scan.setTimeRange(0, Long.MAX_VALUE); - scan.setCaching(1); - ResultScanner scanner = table.getScanner(scan); - int rowNum = scanner.next() != null ? 1 : 0; - // the started scanner shouldn't see the rows put below - for (int i = 1; i < 1000; i++) { - put = new Put(Bytes.toBytes(String.valueOf(i))); - put.setDurability(Durability.ASYNC_WAL); - put.addColumn(FAMILY, Bytes.toBytes(""), Bytes.toBytes(i)); - table.put(put); - } - for (Result result : scanner) { - rowNum++; - } - // scanner should only see two rows - assertEquals(2, rowNum); - scanner = table.getScanner(scan); - rowNum = 0; - for (Result result : scanner) { - rowNum++; - } - // the new scanner should see all rows - assertEquals(1001, rowNum); - } - } - - @Test - public void testPutThenGetWithMultipleThreads() throws Exception { - final int THREAD_NUM = 20; - final int ROUND_NUM = 10; - for (int round = 0; round < ROUND_NUM; round++) { - ArrayList threads = new ArrayList<>(THREAD_NUM); - final AtomicInteger successCnt = new AtomicInteger(0); - try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - for (int i = 0; i < THREAD_NUM; i++) { - final int index = i; - Thread t = new Thread(new Runnable() { - - @Override - public void run() { - final byte[] row = Bytes.toBytes("row-" + index); - final byte[] value = Bytes.toBytes("v" + index); - try { - Put put = new Put(row); - put.addColumn(FAMILY, QUALIFIER, value); - ht.put(put); - Get get = new Get(row); - Result result = ht.get(get); - byte[] returnedValue = result.getValue(FAMILY, QUALIFIER); - if (Bytes.equals(value, returnedValue)) { - successCnt.getAndIncrement(); - } else { - LOG.error("Should be equal but not, original value: " + Bytes.toString(value) - + ", returned value: " - + (returnedValue == null ? "null" : Bytes.toString(returnedValue))); - } - } catch (Throwable e) { - // do nothing - } - } - }); - threads.add(t); - } - for (Thread t : threads) { - t.start(); - } - for (Thread t : threads) { - t.join(); - } - assertEquals("Not equal in round " + round, THREAD_NUM, successCnt.get()); - } - TEST_UTIL.deleteTable(tableName); - } - } - - private static void assertNoLocks(final TableName tableName) - throws IOException, InterruptedException { - HRegion region = (HRegion) find(tableName); - assertEquals(0, region.getLockedRows().size()); - } - - private static HRegion find(final TableName tableName) throws IOException, InterruptedException { - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName); - List regions = rs.getRegions(tableName); - assertEquals(1, regions.size()); - return regions.get(0); - } - - private static T find(final TableName tableName, Class clz) - throws IOException, InterruptedException { - HRegion region = find(tableName); - Coprocessor cp = region.getCoprocessorHost().findCoprocessor(clz.getName()); - assertTrue("The cp instance should be " + clz.getName() + ", current instance is " - + cp.getClass().getName(), clz.isInstance(cp)); - return clz.cast(cp); - } - - public static class WaitingForMultiMutationsObserver - implements RegionCoprocessor, RegionObserver { - final CountDownLatch latch = new CountDownLatch(1); - - @Override - public Optional getRegionObserver() { - return Optional.of(this); - } - - @Override - public void postBatchMutate(final ObserverContext c, - final MiniBatchOperationInProgress miniBatchOp) throws IOException { - try { - latch.await(); - } catch (InterruptedException ex) { - throw new IOException(ex); - } - } - } - - public static class WaitingForScanObserver implements RegionCoprocessor, RegionObserver { - private final CountDownLatch latch = new CountDownLatch(1); - - @Override - public Optional getRegionObserver() { - return Optional.of(this); - } - - @Override - public void postBatchMutate(final ObserverContext c, - final MiniBatchOperationInProgress miniBatchOp) throws IOException { - try { - // waiting for scanner - latch.await(); - } catch (InterruptedException ex) { - throw new IOException(ex); - } - } - - @Override - public RegionScanner postScannerOpen(final ObserverContext e, - final Scan scan, final RegionScanner s) throws IOException { - latch.countDown(); - return s; - } - } - - static byte[] generateHugeValue(int size) { - Random rand = ThreadLocalRandom.current(); - byte[] value = new byte[size]; - for (int i = 0; i < value.length; i++) { - value[i] = (byte) rand.nextInt(256); - } - return value; - } - - @Test - public void testScanWithBatchSizeReturnIncompleteCells() - throws IOException, InterruptedException { - TableDescriptor hd = TableDescriptorBuilder.newBuilder(tableName) - .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setMaxVersions(3).build()) - .build(); - try (Table table = TEST_UTIL.createTable(hd, null)) { - TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS); - - Put put = new Put(ROW); - put.addColumn(FAMILY, Bytes.toBytes(0), generateHugeValue(3 * 1024 * 1024)); - table.put(put); - - put = new Put(ROW); - put.addColumn(FAMILY, Bytes.toBytes(1), generateHugeValue(4 * 1024 * 1024)); - table.put(put); - - for (int i = 2; i < 5; i++) { - for (int version = 0; version < 2; version++) { - put = new Put(ROW); - put.addColumn(FAMILY, Bytes.toBytes(i), generateHugeValue(1024)); - table.put(put); - } - } - - Scan scan = new Scan(); - scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(3) - .setMaxResultSize(4 * 1024 * 1024); - Result result; - try (ResultScanner scanner = table.getScanner(scan)) { - List list = new ArrayList<>(); - /* - * The first scan rpc should return a result with 2 cells, because 3MB + 4MB > 4MB; The - * second scan rpc should return a result with 3 cells, because reach the batch limit = 3; - * The mayHaveMoreCellsInRow in last result should be false in the scan rpc. BTW, the - * moreResultsInRegion also would be false. Finally, the client should collect all the cells - * into two result: 2+3 -> 3+2; - */ - while ((result = scanner.next()) != null) { - list.add(result); - } +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; - Assert.assertEquals(5, list.stream().mapToInt(Result::size).sum()); - Assert.assertEquals(2, list.size()); - Assert.assertEquals(3, list.get(0).size()); - Assert.assertEquals(2, list.get(1).size()); - } +@Tag(LargeTests.TAG) +@Tag(ClientTests.TAG) +public class TestFromClientSide3 extends FromClientSide3TestBase { - scan = new Scan(); - scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(2) - .setMaxResultSize(4 * 1024 * 1024); - try (ResultScanner scanner = table.getScanner(scan)) { - List list = new ArrayList<>(); - while ((result = scanner.next()) != null) { - list.add(result); - } - Assert.assertEquals(5, list.stream().mapToInt(Result::size).sum()); - Assert.assertEquals(3, list.size()); - Assert.assertEquals(2, list.get(0).size()); - Assert.assertEquals(2, list.get(1).size()); - Assert.assertEquals(1, list.get(2).size()); - } - } + @BeforeAll + public static void setUpBeforeAll() throws Exception { + startCluster(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java index 3cc454a8642c..57e21b05e1e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.HConstants.RPC_CODEC_CONF_KEY; -import static org.apache.hadoop.hbase.client.TestFromClientSide3.generateHugeValue; +import static org.apache.hadoop.hbase.client.FromClientSide3TestBase.generateHugeValue; import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_CODEC_CLASS; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtobufRpcServiceImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtobufRpcServiceImpl.java index b2ac0a3deb9b..c38828fa10df 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtobufRpcServiceImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtobufRpcServiceImpl.java @@ -21,6 +21,9 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; @@ -31,7 +34,9 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; @@ -46,7 +51,7 @@ import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos.TestProtobufRpcProto.Interface; @InterfaceAudience.Private -public class TestProtobufRpcServiceImpl implements BlockingInterface { +public class TestProtobufRpcServiceImpl implements BlockingInterface, Interface { public static final BlockingService SERVICE = TestProtobufRpcProto.newReflectiveBlockingService(new TestProtobufRpcServiceImpl()); @@ -119,4 +124,63 @@ public AddrResponseProto addr(RpcController controller, EmptyRequestProto reques return AddrResponseProto.newBuilder() .setAddr(RpcServer.getRemoteAddress().get().getHostAddress()).build(); } + + @Override + public void ping(RpcController controller, EmptyRequestProto request, + RpcCallback done) { + done.run(EmptyResponseProto.getDefaultInstance()); + } + + @Override + public void echo(RpcController controller, EchoRequestProto request, + RpcCallback done) { + if (controller instanceof HBaseRpcController) { + HBaseRpcController pcrc = (HBaseRpcController) controller; + // If cells, scan them to check we are able to iterate what we were given and since this is an + // echo, just put them back on the controller creating a new block. Tests our block building. + CellScanner cellScanner = pcrc.cellScanner(); + List list = null; + if (cellScanner != null) { + list = new ArrayList<>(); + try { + while (cellScanner.advance()) { + list.add(cellScanner.current()); + } + } catch (IOException e) { + pcrc.setFailed(e); + return; + } + } + cellScanner = CellUtil.createCellScanner(list); + pcrc.setCellScanner(cellScanner); + } + done.run(EchoResponseProto.newBuilder().setMessage(request.getMessage()).build()); + } + + @Override + public void error(RpcController controller, EmptyRequestProto request, + RpcCallback done) { + if (controller instanceof HBaseRpcController) { + ((HBaseRpcController) controller).setFailed(new DoNotRetryIOException("server error!")); + } else { + controller.setFailed("server error!"); + } + } + + private final ScheduledExecutorService executor = + Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).build()); + + @Override + public void pause(RpcController controller, PauseRequestProto request, + RpcCallback done) { + executor.schedule(() -> done.run(EmptyResponseProto.getDefaultInstance()), request.getMs(), + TimeUnit.MILLISECONDS); + } + + @Override + public void addr(RpcController controller, EmptyRequestProto request, + RpcCallback done) { + done.run(AddrResponseProto.newBuilder() + .setAddr(RpcServer.getRemoteAddress().get().getHostAddress()).build()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java index 13b73a0d104a..a00c218253a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java @@ -22,8 +22,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; import java.io.IOException; @@ -52,6 +53,12 @@ import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.AddrResponseProto; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoRequestProto; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoResponseProto; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EmptyRequestProto; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EmptyResponseProto; +import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.PauseRequestProto; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; @@ -111,7 +118,41 @@ public DummyCpService() { @Override public Iterable getServices() { - return Collections.singleton(mock(TestRpcServiceProtos.TestProtobufRpcProto.class)); + return Collections.singleton( + TestRpcServiceProtos.TestProtobufRpcProto.newReflectiveService(new DummyService())); + } + } + + private static final class DummyService + implements TestRpcServiceProtos.TestProtobufRpcProto.Interface { + + @Override + public void ping(RpcController controller, EmptyRequestProto request, + RpcCallback done) { + + } + + @Override + public void echo(RpcController controller, EchoRequestProto request, + RpcCallback done) { + + } + + @Override + public void error(RpcController controller, EmptyRequestProto request, + RpcCallback done) { + + } + + @Override + public void pause(RpcController controller, PauseRequestProto request, + RpcCallback done) { + + } + + @Override + public void addr(RpcController controller, EmptyRequestProto request, + RpcCallback done) { } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFromClientSide3WoUnsafe.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFromClientSide3WoUnsafe.java index 71f3c3ee7924..ffec01edafd7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFromClientSide3WoUnsafe.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFromClientSide3WoUnsafe.java @@ -17,31 +17,29 @@ */ package org.apache.hadoop.hbase.util; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.client.TestFromClientSide3; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mockStatic; + +import org.apache.hadoop.hbase.client.FromClientSide3TestBase; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.experimental.categories.Category; - -@Category({ LargeTests.class, ClientTests.class }) -public class TestFromClientSide3WoUnsafe extends TestFromClientSide3 { +import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.mockito.MockedStatic; - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestFromClientSide3WoUnsafe.class); - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TestByteBufferUtils.disableUnsafe(); - TestFromClientSide3.setUpBeforeClass(); - } +@Tag(LargeTests.TAG) +@Tag(ClientTests.TAG) +public class TestFromClientSide3WoUnsafe extends FromClientSide3TestBase { - @AfterClass - public static void tearDownAfterClass() throws Exception { - TestFromClientSide3.tearDownAfterClass(); - TestByteBufferUtils.detectAvailabilityOfUnsafe(); + @BeforeAll + public static void setUpBeforeAll() throws Exception { + try (MockedStatic mocked = mockStatic(HBasePlatformDependent.class)) { + mocked.when(HBasePlatformDependent::isUnsafeAvailable).thenReturn(false); + mocked.when(HBasePlatformDependent::unaligned).thenReturn(false); + assertFalse(ByteBufferUtils.UNSAFE_AVAIL); + assertFalse(ByteBufferUtils.UNSAFE_UNALIGNED); + } + startCluster(); } } From 11bc9e917f86ea59f51e978aa34eab55f7bc8523 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 9 Oct 2025 06:21:05 +0200 Subject: [PATCH 081/336] HBASE-29649 Un-deprecate preWALRestore and postWALRestore in RegionCoprocessorHost (#7370) Signed-off-by: Duo Zhang (cherry picked from commit 361a563addd1dfc1ff866c2fdcb4e15233886e54) --- .../hadoop/hbase/regionserver/RegionCoprocessorHost.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 52b3b54f4b24..3a59cc863307 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -1440,10 +1440,7 @@ public void call(RegionObserver observer) throws IOException { /** * Supports Coprocessor 'bypass'. * @return true if default behavior should be bypassed, false otherwise - * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced with - * something that doesn't expose IntefaceAudience.Private classes. */ - @Deprecated public boolean preWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit) throws IOException { return execOperation( @@ -1455,11 +1452,6 @@ public void call(RegionObserver observer) throws IOException { }); } - /** - * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced with - * something that doesn't expose IntefaceAudience.Private classes. - */ - @Deprecated public void postWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new RegionObserverOperationWithoutResult() { From 5f63495fa9a33459444716772714c4a7dd4293f8 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 9 Oct 2025 16:46:18 +0800 Subject: [PATCH 082/336] HBASE-29637 Implement ResourceCheckerJUnitListener for junit 5 (#7366) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Istvan Toth (cherry picked from commit d8b1912361218ce9de3c3f7d4c5194c23519fa69) --- .../hadoop/hbase/HBaseJupiterExtension.java | 63 +++++--- .../hadoop/hbase/JUnitResourceCheckers.java | 141 ++++++++++++++++++ .../hbase/ResourceCheckerJUnitListener.java | 114 +------------- 3 files changed, 189 insertions(+), 129 deletions(-) create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java index 997e3dfa357a..9d4ea87e0ec1 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java @@ -37,7 +37,9 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.yetus.audience.InterfaceAudience; import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Store; import org.junit.jupiter.api.extension.InvocationInterceptor; @@ -60,14 +62,18 @@ * the tag. *

* It also controls the timeout for the whole test class running, while the timeout annotation in - * JUnit5 can only enforce the timeout for each test method. + * JUnit5 can only enforce the timeout for each test method. When a test is timed out, a thread dump + * will be printed to log output. *

- * Finally, it also forbid System.exit call in tests. TODO: need to find a new way as - * SecurityManager has been removed since Java 21. + * It also implements resource check for each test method, using the {@link ResourceChecker} class. + *

+ * Finally, it also forbid System.exit call in tests.
+ * TODO: need to find a new way as SecurityManager was deprecated in Java 17 and permanently + * disabled since Java 24. */ @InterfaceAudience.Private -public class HBaseJupiterExtension - implements InvocationInterceptor, BeforeAllCallback, AfterAllCallback { +public class HBaseJupiterExtension implements InvocationInterceptor, BeforeAllCallback, + AfterAllCallback, BeforeEachCallback, AfterEachCallback { private static final Logger LOG = LoggerFactory.getLogger(HBaseJupiterExtension.class); @@ -84,6 +90,8 @@ public class HBaseJupiterExtension private static final String DEADLINE = "deadline"; + private static final String RESOURCE_CHECK = "rc"; + private Duration pickTimeout(ExtensionContext ctx) { Set timeoutTags = TAG_TO_TIMEOUT.keySet(); Set timeoutTag = Sets.intersection(timeoutTags, ctx.getTags()); @@ -130,7 +138,8 @@ public void afterAll(ExtensionContext ctx) throws Exception { System.setSecurityManager(null); } - private T runWithTimeout(Invocation invocation, ExtensionContext ctx) throws Throwable { + private T runWithTimeout(Invocation invocation, ExtensionContext ctx, String name) + throws Throwable { Store store = ctx.getStore(NAMESPACE); ExecutorService executor = store.get(EXECUTOR, ExecutorService.class); if (executor == null) { @@ -139,12 +148,12 @@ private T runWithTimeout(Invocation invocation, ExtensionContext ctx) thr Instant deadline = store.get(DEADLINE, Instant.class); Instant now = Instant.now(); if (!now.isBefore(deadline)) { - fail("Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline); + fail("Test " + name + " timed out, deadline is " + deadline); return null; } Duration remaining = Duration.between(now, deadline); - LOG.info("remaining timeout for {} is {}", ctx.getDisplayName(), remaining); + LOG.info("remaining timeout for {} is {}", name, remaining); Future future = executor.submit(() -> { try { return invocation.proceed(); @@ -157,14 +166,13 @@ private T runWithTimeout(Invocation invocation, ExtensionContext ctx) thr return future.get(remaining.toNanos(), TimeUnit.NANOSECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - fail("Test " + ctx.getDisplayName() + " interrupted"); + fail("Test " + name + " interrupted"); return null; } catch (ExecutionException e) { throw ExceptionUtils.throwAsUncheckedException(e.getCause()); } catch (TimeoutException e) { printThreadDump(); - throw new JUnitException( - "Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline, e); + throw new JUnitException("Test " + name + " timed out, deadline is " + deadline, e); } } @@ -177,41 +185,62 @@ private void printThreadDump() { public void interceptBeforeAllMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeAll"); } @Override public void interceptBeforeEachMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeEach"); } @Override public void interceptTestMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName()); } @Override public void interceptAfterEachMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterEach"); } @Override public void interceptAfterAllMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterAll"); } @Override public T interceptTestClassConstructor(Invocation invocation, ReflectiveInvocationContext> invocationContext, ExtensionContext extensionContext) throws Throwable { - return runWithTimeout(invocation, extensionContext); + return runWithTimeout(invocation, extensionContext, + extensionContext.getDisplayName() + ".constructor"); + } + + // below are for implementing resource checker around test method + + @Override + public void beforeEach(ExtensionContext ctx) throws Exception { + ResourceChecker rc = new ResourceChecker(ctx.getDisplayName()); + JUnitResourceCheckers.addResourceAnalyzer(rc); + Store store = ctx.getStore(NAMESPACE); + store.put(RESOURCE_CHECK, rc); + rc.start(); + } + + @Override + public void afterEach(ExtensionContext ctx) throws Exception { + Store store = ctx.getStore(NAMESPACE); + ResourceChecker rc = store.remove(RESOURCE_CHECK, ResourceChecker.class); + if (rc != null) { + rc.end(); + } } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java new file mode 100644 index 000000000000..aee49dc2c60f --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.ResourceChecker.Phase; +import org.apache.hadoop.hbase.util.JVM; + +/** + * ResourceCheckers when running JUnit tests. + */ +public final class JUnitResourceCheckers { + + private JUnitResourceCheckers() { + } + + private static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + private Set initialThreadNames = new HashSet<>(); + private List stringsToLog = null; + + @Override + public int getVal(Phase phase) { + Map stackTraces = Thread.getAllStackTraces(); + if (phase == Phase.INITIAL) { + stringsToLog = null; + for (Thread t : stackTraces.keySet()) { + initialThreadNames.add(t.getName()); + } + } else if (phase == Phase.END) { + if (stackTraces.size() > initialThreadNames.size()) { + stringsToLog = new ArrayList<>(); + for (Thread t : stackTraces.keySet()) { + if (!initialThreadNames.contains(t.getName())) { + stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); + StackTraceElement[] stackElements = stackTraces.get(t); + for (StackTraceElement ele : stackElements) { + stringsToLog.add("\t" + ele + "\n"); + } + } + } + } + } + return stackTraces.size(); + } + + @Override + public int getMax() { + return 500; + } + + @Override + public List getStringsToLog() { + return stringsToLog; + } + } + + private static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getOpenFileDescriptorCount(); + } + + @Override + public int getMax() { + return 1024; + } + } + + private static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getMaxFileDescriptorCount(); + } + } + + private static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getSystemLoadAverage() * 100); + } + } + + private static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return new JVM().getNumberOfRunningProcess(); + } + } + + private static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); + } + } + + public static void addResourceAnalyzer(ResourceChecker rc) { + rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); + rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); + rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); + rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java index 4dfce7f536b5..2a796cc40774 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java @@ -17,14 +17,8 @@ */ package org.apache.hadoop.hbase; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hbase.ResourceChecker.Phase; -import org.apache.hadoop.hbase.util.JVM; import org.junit.runner.notification.RunListener; /** @@ -38,104 +32,8 @@ * When surefire forkMode=once/always/perthread, this code is executed on the forked process. */ public class ResourceCheckerJUnitListener extends RunListener { - private Map rcs = new ConcurrentHashMap<>(); - static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - private static Set initialThreadNames = new HashSet<>(); - private static List stringsToLog = null; - - @Override - public int getVal(Phase phase) { - Map stackTraces = Thread.getAllStackTraces(); - if (phase == Phase.INITIAL) { - stringsToLog = null; - for (Thread t : stackTraces.keySet()) { - initialThreadNames.add(t.getName()); - } - } else if (phase == Phase.END) { - if (stackTraces.size() > initialThreadNames.size()) { - stringsToLog = new ArrayList<>(); - for (Thread t : stackTraces.keySet()) { - if (!initialThreadNames.contains(t.getName())) { - stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); - StackTraceElement[] stackElements = stackTraces.get(t); - for (StackTraceElement ele : stackElements) { - stringsToLog.add("\t" + ele + "\n"); - } - } - } - } - } - return stackTraces.size(); - } - - @Override - public int getMax() { - return 500; - } - - @Override - public List getStringsToLog() { - return stringsToLog; - } - } - - static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getOpenFileDescriptorCount(); - } - - @Override - public int getMax() { - return 1024; - } - } - - static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getMaxFileDescriptorCount(); - } - } - - static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getSystemLoadAverage() * 100); - } - } - - static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return new JVM().getNumberOfRunningProcess(); - } - } - - static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); - } - } + private final Map rcs = new ConcurrentHashMap<>(); /** * To be implemented by sub classes if they want to add specific ResourceAnalyzer. @@ -145,17 +43,9 @@ protected void addResourceAnalyzer(ResourceChecker rc) { private void start(String testName) { ResourceChecker rc = new ResourceChecker(testName); - rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); - rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); - rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); - rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); - + JUnitResourceCheckers.addResourceAnalyzer(rc); addResourceAnalyzer(rc); - rcs.put(testName, rc); - rc.start(); } From 096834d1ac6581f3358da23b4099fbff142eb2ed Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Fri, 10 Oct 2025 16:30:41 +0200 Subject: [PATCH 083/336] HBASE-29650 Upgrade tomcat-jasper to 9.0.110 (#7373) Signed-off-by: Duo Zhang (cherry picked from commit dfeddb39ffad78ac5f42f66d3aa8e341a4ba1a51) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b57dcdcd8ec..0c41eda95d92 100644 --- a/pom.xml +++ b/pom.xml @@ -594,7 +594,7 @@ 2.3.1 3.1.0 2.1.1 - 9.0.104 + 9.0.110 9.3.15.0 5.13.4 5.13.4 From 2261441c171df439d84e660dfef64767b8067039 Mon Sep 17 00:00:00 2001 From: DieterDP <90392398+DieterDP-ng@users.noreply.github.com> Date: Fri, 10 Oct 2025 11:56:56 +0200 Subject: [PATCH 084/336] HBASE-29604 BackupHFileCleaner uses flawed time based check (#7360) Adds javadoc mentioning the concurrent usage and thread-safety need of FileCleanerDelegate#getDeletableFiles. Fixes a potential thread-safety issue in BackupHFileCleaner: this class tracks timestamps to block the deletion of recently loaded HFiles that might be needed for backup purposes. The timestamps were being registered from inside the concurrent method, which could result in recently added files getting deleted. Moved the timestamp registration to the postClean method, which is called only a single time per cleaner run, so recently loaded HFiles are in fact protected from deletion. Signed-off-by: Nick Dimiduk --- .../hadoop/hbase/backup/BackupHFileCleaner.java | 17 ++++++++++------- .../hbase/backup/TestBackupHFileCleaner.java | 13 ++++++++++--- .../master/cleaner/BaseFileCleanerDelegate.java | 4 ++++ .../master/cleaner/FileCleanerDelegate.java | 4 ++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java index c9a76bef2891..bbbae2d631fe 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java @@ -52,10 +52,13 @@ public class BackupHFileCleaner extends BaseHFileCleanerDelegate implements Abor private boolean stopped = false; private boolean aborted = false; private Connection connection; - // timestamp of most recent read from backup system table - private long prevReadFromBackupTbl = 0; - // timestamp of 2nd most recent read from backup system table - private long secondPrevReadFromBackupTbl = 0; + // timestamp of most recent completed cleaning run + private volatile long previousCleaningCompletionTimestamp = 0; + + @Override + public void postClean() { + previousCleaningCompletionTimestamp = EnvironmentEdgeManager.currentTime(); + } @Override public Iterable getDeletableFiles(Iterable files) { @@ -79,12 +82,12 @@ public Iterable getDeletableFiles(Iterable files) { return Collections.emptyList(); } - secondPrevReadFromBackupTbl = prevReadFromBackupTbl; - prevReadFromBackupTbl = EnvironmentEdgeManager.currentTime(); + // Pin the threshold, we don't want the result to change depending on evaluation time. + final long recentFileThreshold = previousCleaningCompletionTimestamp; return Iterables.filter(files, file -> { // If the file is recent, be conservative and wait for one more scan of the bulk loads - if (file.getModificationTime() > secondPrevReadFromBackupTbl) { + if (file.getModificationTime() > recentFileThreshold) { LOG.debug("Preventing deletion due to timestamp: {}", file.getPath().toString()); return false; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java index bfc729aa6792..3c5cff9fa63f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java @@ -108,11 +108,11 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { Iterable deletable; // The first call will not allow any deletions because of the timestamp mechanism. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Collections.emptySet(), Sets.newHashSet(deletable)); // No bulk loads registered, so all files can be deleted. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file1, file1Archived, file2, file3), Sets.newHashSet(deletable)); // Register some bulk loads. @@ -125,10 +125,17 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { } // File 1 can no longer be deleted, because it is registered as a bulk load. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file2, file3), Sets.newHashSet(deletable)); } + private Iterable callCleaner(BackupHFileCleaner cleaner, Iterable files) { + cleaner.preClean(); + Iterable deletable = cleaner.getDeletableFiles(files); + cleaner.postClean(); + return deletable; + } + private FileStatus createFile(String fileName) throws IOException { Path file = new Path(root, fileName); fs.createNewFile(file); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java index 4c24ba1f81c5..700914f07b90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java @@ -44,6 +44,10 @@ public void init(Map params) { /** * Should the master delete the file or keep it? + *

+ * This method can be called concurrently by multiple threads. Implementations must be thread + * safe. + *

* @param fStat file status of the file to check * @return true if the file is deletable, false if not */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java index d37bb6202730..438f34a891ce 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java @@ -33,6 +33,10 @@ public interface FileCleanerDelegate extends Configurable, Stoppable { /** * Determines which of the given files are safe to delete + *

+ * This method can be called concurrently by multiple threads. Implementations must be thread + * safe. + *

* @param files files to check for deletion * @return files that are ok to delete according to this cleaner */ From 202e5e04eaa7ba5d02834e85a76bba6c7dc60a14 Mon Sep 17 00:00:00 2001 From: gong-flying <106514313+gong-flying@users.noreply.github.com> Date: Wed, 15 Oct 2025 15:53:34 +0800 Subject: [PATCH 085/336] HBASE-29653 Upgrade os-maven-plugin to 1.7.1 for RISC-V riscv64 support (#7376) Signed-off-by: Istvan Toth Signed-off-by: Duo Zhang (cherry picked from commit bab3df9cff68d4569367b522cbb6e44fd68057b4) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c41eda95d92..8d048c479fd8 100644 --- a/pom.xml +++ b/pom.xml @@ -655,7 +655,7 @@ 1.1.0 3.1.2 12.1.0 - 1.5.0.Final + 1.7.1 1.3.9-1 4.7.3 4.7.2.1 From 4db0e0b58e76085065c778caa44120605441dcc9 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Thu, 16 Oct 2025 13:23:30 -0400 Subject: [PATCH 086/336] HBASE-29631 Fix race condition in IncrementalTableBackupClient when HFiles are archived during backup (#7346) (#7357) (#7359) Signed-off-by: Ray Mattingly Co-authored-by: Siddharth Khillon Co-authored-by: Hernan Romer Co-authored-by: skhillon --- .../impl/IncrementalTableBackupClient.java | 23 +- .../TestIncrementalBackupWithBulkLoad.java | 262 ++++++++++++++++++ 2 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index ae32a8dbeb51..e5599c8357cc 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -200,6 +200,9 @@ private void mergeSplitAndCopyBulkloadedHFiles(List activeFiles, int numActiveFiles = activeFiles.size(); updateFileLists(activeFiles, archiveFiles); if (activeFiles.size() < numActiveFiles) { + // We've archived some files, delete bulkloads directory + // and re-try + deleteBulkLoadDirectory(); continue; } @@ -242,7 +245,7 @@ private void mergeSplitAndCopyBulkloadedHFiles(List files, TableName tn, incrementalCopyBulkloadHFiles(tgtFs, tn); } - private void updateFileLists(List activeFiles, List archiveFiles) + public void updateFileLists(List activeFiles, List archiveFiles) throws IOException { List newlyArchived = new ArrayList<>(); @@ -252,9 +255,23 @@ private void updateFileLists(List activeFiles, List archiveFiles } } - if (newlyArchived.size() > 0) { + if (!newlyArchived.isEmpty()) { + String rootDir = CommonFSUtils.getRootDir(conf).toString(); + activeFiles.removeAll(newlyArchived); - archiveFiles.addAll(newlyArchived); + for (String file : newlyArchived) { + String archivedFile = file.substring(rootDir.length() + 1); + Path archivedFilePath = new Path(HFileArchiveUtil.getArchivePath(conf), archivedFile); + archivedFile = archivedFilePath.toString(); + + if (!fs.exists(archivedFilePath)) { + throw new IOException(String.format( + "File %s no longer exists, and no archived file %s exists for it", file, archivedFile)); + } + + LOG.debug("Archived file {} has been updated", archivedFile); + archiveFiles.add(archivedFile); + } } LOG.debug(newlyArchived.size() + " files have been archived."); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java new file mode 100644 index 000000000000..a2ca4304174c --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.impl.BulkLoad; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.tool.BulkLoadHFiles; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.HFileTestUtil; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + +/** + * This test checks whether backups properly track & manage bulk files loads. + */ +@Category(LargeTests.class) +public class TestIncrementalBackupWithBulkLoad extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestIncrementalBackupWithBulkLoad.class); + + private static final String TEST_NAME = TestIncrementalBackupWithBulkLoad.class.getSimpleName(); + private static final int ROWS_IN_BULK_LOAD = 100; + + // implement all test cases in 1 test since incremental backup/restore has dependencies + @Test + public void TestIncBackupDeleteTable() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // The test starts with some data, and no bulk loaded rows. + int expectedRowCount = NB_ROWS_IN_BATCH; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertTrue(systemTable.readBulkloadRows(ImmutableList.of(table1)).isEmpty()); + + // Bulk loads aren't tracked if the table isn't backed up yet + performBulkLoad("bulk1"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Create a backup, bulk loads are now being tracked + String backup1 = backupTables(BackupType.FULL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup1)); + performBulkLoad("bulk2"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(1, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Truncating or deleting a table clears the tracked bulk loads (and all rows) + TEST_UTIL.truncateTable(table1).close(); + expectedRowCount = 0; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Creating a full backup clears the bulk loads (since they are captured in the snapshot) + performBulkLoad("bulk3"); + expectedRowCount = ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(1, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + String backup2 = backupTables(BackupType.FULL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup2)); + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + + // Creating an incremental backup clears the bulk loads + performBulkLoad("bulk4"); + performBulkLoad("bulk5"); + performBulkLoad("bulk6"); + expectedRowCount += 3 * ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(3, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + String backup3 = + backupTables(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR); + assertTrue(checkSucceeded(backup3)); + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + assertEquals(0, systemTable.readBulkloadRows(ImmutableList.of(table1)).size()); + int rowCountAfterBackup3 = expectedRowCount; + + // Doing another bulk load, to check that this data will disappear after a restore operation + performBulkLoad("bulk7"); + expectedRowCount += ROWS_IN_BULK_LOAD; + assertEquals(expectedRowCount, TEST_UTIL.countRows(table1)); + List bulkloadsTemp = systemTable.readBulkloadRows(ImmutableList.of(table1)); + assertEquals(1, bulkloadsTemp.size()); + BulkLoad bulk7 = bulkloadsTemp.get(0); + + // Doing a restore. Overwriting the table implies clearing the bulk loads, + // but the loading of restored data involves loading bulk data, we expect 2 bulk loads + // associated with backup 3 (loading of full backup, loading of incremental backup). + BackupAdmin client = getBackupAdmin(); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backup3, false, + new TableName[] { table1 }, new TableName[] { table1 }, true)); + assertEquals(rowCountAfterBackup3, TEST_UTIL.countRows(table1)); + List bulkLoads = systemTable.readBulkloadRows(ImmutableList.of(table1)); + assertEquals(2, bulkLoads.size()); + assertFalse(bulkLoads.contains(bulk7)); + + // Check that we have data of all expected bulk loads + try (Table restoredTable = TEST_UTIL.getConnection().getTable(table1)) { + assertFalse(containsRowWithKey(restoredTable, "bulk1")); + assertFalse(containsRowWithKey(restoredTable, "bulk2")); + assertTrue(containsRowWithKey(restoredTable, "bulk3")); + assertTrue(containsRowWithKey(restoredTable, "bulk4")); + assertTrue(containsRowWithKey(restoredTable, "bulk5")); + assertTrue(containsRowWithKey(restoredTable, "bulk6")); + assertFalse(containsRowWithKey(restoredTable, "bulk7")); + } + } + } + + private boolean containsRowWithKey(Table table, String rowKey) throws IOException { + byte[] data = Bytes.toBytes(rowKey); + Get get = new Get(data); + Result result = table.get(get); + return result.containsColumn(famName, qualName); + } + + @Test + public void testUpdateFileListsRaceCondition() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // Test the race condition where files are archived during incremental backup + FileSystem fs = TEST_UTIL.getTestFileSystem(); + + String regionName = "region1"; + String columnFamily = "cf"; + String filename1 = "hfile1"; + String filename2 = "hfile2"; + + Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table1); + Path activeFile1 = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename1); + Path activeFile2 = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename2); + + fs.mkdirs(activeFile1.getParent()); + fs.create(activeFile1).close(); + fs.create(activeFile2).close(); + + List activeFiles = new ArrayList<>(); + activeFiles.add(activeFile1.toString()); + activeFiles.add(activeFile2.toString()); + List archiveFiles = new ArrayList<>(); + + Path archiveDir = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), table1, + regionName, columnFamily); + Path archivedFile1 = new Path(archiveDir, filename1); + fs.mkdirs(archiveDir); + assertTrue("File should be moved to archive", fs.rename(activeFile1, archivedFile1)); + + TestBackupBase.IncrementalTableBackupClientForTest client = + new TestBackupBase.IncrementalTableBackupClientForTest(TEST_UTIL.getConnection(), + "test_backup_id", + createBackupRequest(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR)); + + client.updateFileLists(activeFiles, archiveFiles); + + assertEquals("Only one file should remain in active files", 1, activeFiles.size()); + assertEquals("File2 should still be in active files", activeFile2.toString(), + activeFiles.get(0)); + assertEquals("One file should be added to archive files", 1, archiveFiles.size()); + assertEquals("Archived file should have correct path", archivedFile1.toString(), + archiveFiles.get(0)); + systemTable.finishBackupExclusiveOperation(); + } + + } + + @Test + public void testUpdateFileListsMissingArchivedFile() throws Exception { + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + // Test that IOException is thrown when file doesn't exist in archive location + FileSystem fs = TEST_UTIL.getTestFileSystem(); + + String regionName = "region2"; + String columnFamily = "cf"; + String filename = "missing_file"; + + Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration()); + Path tableDir = CommonFSUtils.getTableDir(rootDir, table1); + Path activeFile = + new Path(tableDir, regionName + Path.SEPARATOR + columnFamily + Path.SEPARATOR + filename); + + fs.mkdirs(activeFile.getParent()); + fs.create(activeFile).close(); + + List activeFiles = new ArrayList<>(); + activeFiles.add(activeFile.toString()); + List archiveFiles = new ArrayList<>(); + + // Delete the file but don't create it in archive location + fs.delete(activeFile, false); + + TestBackupBase.IncrementalTableBackupClientForTest client = + new TestBackupBase.IncrementalTableBackupClientForTest(TEST_UTIL.getConnection(), + "test_backup_id", + createBackupRequest(BackupType.INCREMENTAL, ImmutableList.of(table1), BACKUP_ROOT_DIR)); + + // This should throw IOException since file doesn't exist in archive + try { + client.updateFileLists(activeFiles, archiveFiles); + fail("Expected IOException to be thrown"); + } catch (IOException e) { + // Expected + } + systemTable.finishBackupExclusiveOperation(); + } + } + + private void performBulkLoad(String keyPrefix) throws IOException { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Path baseDirectory = TEST_UTIL.getDataTestDirOnTestFS(TEST_NAME); + Path hfilePath = + new Path(baseDirectory, Bytes.toString(famName) + Path.SEPARATOR + "hfile_" + keyPrefix); + + HFileTestUtil.createHFile(TEST_UTIL.getConfiguration(), fs, hfilePath, famName, qualName, + Bytes.toBytes(keyPrefix), Bytes.toBytes(keyPrefix + "z"), ROWS_IN_BULK_LOAD); + + Map result = + BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(table1, baseDirectory); + assertFalse(result.isEmpty()); + } +} From 26cbc3e344702be344cec47dfba99d763f8ea9ce Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 17 Oct 2025 10:51:32 -0400 Subject: [PATCH 087/336] HBASE-29663 TimeBasedLimiters should support dynamic configuration refresh (#7387) (#7393) (#7395) Signed-off-by: Charles Connell Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../quotas/FixedIntervalRateLimiter.java | 18 ++++++--- .../hadoop/hbase/quotas/QuotaCache.java | 13 ++++--- .../hbase/quotas/QuotaLimiterFactory.java | 5 ++- .../hadoop/hbase/quotas/QuotaState.java | 5 ++- .../apache/hadoop/hbase/quotas/QuotaUtil.java | 38 ++++++++++--------- .../hadoop/hbase/quotas/TimeBasedLimiter.java | 8 ++-- .../hadoop/hbase/quotas/UserQuotaState.java | 19 +++++----- .../TestRegionCoprocessorQuotaUsage.java | 12 +++++- .../quotas/TestDefaultOperationQuota.java | 16 ++++---- .../hadoop/hbase/quotas/TestQuotaCache2.java | 12 ++++-- .../hadoop/hbase/quotas/TestQuotaState.java | 22 ++++++----- 11 files changed, 99 insertions(+), 69 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java index c5b2fc7f5d83..a71b5d4b2fba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java @@ -20,8 +20,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; - -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * With this limiter resources will be refilled only after a fixed interval of time. @@ -43,6 +43,8 @@ public class FixedIntervalRateLimiter extends RateLimiter { public static final String RATE_LIMITER_REFILL_INTERVAL_MS = "hbase.quota.rate.limiter.refill.interval.ms"; + private static final Logger LOG = LoggerFactory.getLogger(FixedIntervalRateLimiter.class); + private long nextRefillTime = -1L; private final long refillInterval; @@ -52,10 +54,14 @@ public FixedIntervalRateLimiter() { public FixedIntervalRateLimiter(long refillInterval) { super(); - Preconditions.checkArgument(getTimeUnitInMillis() >= refillInterval, - String.format("Refill interval %s must be less than or equal to TimeUnit millis %s", - refillInterval, getTimeUnitInMillis())); - this.refillInterval = refillInterval; + long timeUnit = getTimeUnitInMillis(); + if (refillInterval > timeUnit) { + LOG.warn( + "Refill interval {} is larger than time unit {}. This is invalid. " + + "Instead, we will use the time unit {} as the refill interval", + refillInterval, timeUnit, timeUnit); + } + this.refillInterval = Math.min(timeUnit, refillInterval); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 325f31586c5e..c95578dc5d00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -130,20 +130,23 @@ private void ensureInitialized() { } private Map fetchUserQuotaStateEntries() throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), tableMachineQuotaFactors, - machineQuotaFactor); + return QuotaUtil.fetchUserQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + tableMachineQuotaFactors, machineQuotaFactor); } private Map fetchRegionServerQuotaStateEntries() throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection()); + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConfiguration(), + rsServices.getConnection()); } private Map fetchTableQuotaStateEntries() throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), tableMachineQuotaFactors); + return QuotaUtil.fetchTableQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + tableMachineQuotaFactors); } private Map fetchNamespaceQuotaStateEntries() throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), machineQuotaFactor); + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + machineQuotaFactor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java index 762896773fc7..63d8df65d25d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -25,8 +26,8 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class QuotaLimiterFactory { - public static QuotaLimiter fromThrottle(final Throttle throttle) { - return TimeBasedLimiter.fromThrottle(throttle); + public static QuotaLimiter fromThrottle(Configuration conf, final Throttle throttle) { + return TimeBasedLimiter.fromThrottle(conf, throttle); } public static QuotaLimiter update(final QuotaLimiter a, final QuotaLimiter b) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java index 61aa9d7f068f..4a0b634abec5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -57,9 +58,9 @@ public synchronized boolean isBypass() { /** * Setup the global quota information. (This operation is part of the QuotaState setup) */ - public synchronized void setQuotas(final Quotas quotas) { + public synchronized void setQuotas(Configuration conf, final Quotas quotas) { if (quotas.hasThrottle()) { - globalLimiter = QuotaLimiterFactory.fromThrottle(quotas.getThrottle()); + globalLimiter = QuotaLimiterFactory.fromThrottle(conf, quotas.getThrottle()); } else { globalLimiter = NoopQuotaLimiter.get(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 6b38635eccc0..f7df09801e0f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -329,8 +329,9 @@ private static void deleteQuotas(final Connection connection, final byte[] rowKe doDelete(connection, delete); } - public static Map fetchUserQuotas(final Connection connection, - Map tableMachineQuotaFactors, double factor) throws IOException { + public static Map fetchUserQuotas(final Configuration conf, + final Connection connection, Map tableMachineQuotaFactors, double factor) + throws IOException { Map userQuotas = new HashMap<>(); try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { Scan scan = new Scan(); @@ -350,7 +351,7 @@ public static Map fetchUserQuotas(final Connection conne @Override public void visitUserQuotas(String userName, String namespace, Quotas quotas) { quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(namespace, quotas); + quotaInfo.setQuotas(conf, namespace, quotas); } @Override @@ -359,13 +360,13 @@ public void visitUserQuotas(String userName, TableName table, Quotas quotas) { tableMachineQuotaFactors.containsKey(table) ? tableMachineQuotaFactors.get(table) : 1); - quotaInfo.setQuotas(table, quotas); + quotaInfo.setQuotas(conf, table, quotas); } @Override public void visitUserQuotas(String userName, Quotas quotas) { quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(quotas); + quotaInfo.setQuotas(conf, quotas); } }); } catch (IOException e) { @@ -406,7 +407,7 @@ protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) { UserQuotaState state = new UserQuotaState(); QuotaProtos.Quotas defaultQuotas = QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build(); - state.setQuotas(defaultQuotas); + state.setQuotas(conf, defaultQuotas); return state; } @@ -419,12 +420,12 @@ private static Optional buildDefaultTimedQuota(Configuration conf, S java.util.concurrent.TimeUnit.SECONDS, org.apache.hadoop.hbase.quotas.QuotaScope.MACHINE)); } - public static Map fetchTableQuotas(final Connection connection, - Map tableMachineFactors) throws IOException { + public static Map fetchTableQuotas(final Configuration conf, + final Connection connection, Map tableMachineFactors) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_TABLE_ROW_KEY_PREFIX); - return fetchGlobalQuotas("table", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "table", scan, connection, new KeyFromRow() { @Override public TableName getKeyFromRow(final byte[] row) { assert isTableRowKey(row); @@ -438,12 +439,12 @@ public double getFactor(TableName tableName) { }); } - public static Map fetchNamespaceQuotas(final Connection connection, - double factor) throws IOException { + public static Map fetchNamespaceQuotas(final Configuration conf, + final Connection connection, double factor) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_NAMESPACE_ROW_KEY_PREFIX); - return fetchGlobalQuotas("namespace", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "namespace", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isNamespaceRowKey(row); @@ -457,12 +458,12 @@ public double getFactor(String s) { }); } - public static Map fetchRegionServerQuotas(final Connection connection) - throws IOException { + public static Map fetchRegionServerQuotas(final Configuration conf, + final Connection connection) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_REGION_SERVER_ROW_KEY_PREFIX); - return fetchGlobalQuotas("regionServer", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "regionServer", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isRegionServerRowKey(row); @@ -476,8 +477,9 @@ public double getFactor(String s) { }); } - public static Map fetchGlobalQuotas(final String type, final Scan scan, - final Connection connection, final KeyFromRow kfr) throws IOException { + public static Map fetchGlobalQuotas(final Configuration conf, + final String type, final Scan scan, final Connection connection, final KeyFromRow kfr) + throws IOException { Map globalQuotas = new HashMap<>(); try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { @@ -498,7 +500,7 @@ public static Map fetchGlobalQuotas(final String type, final try { Quotas quotas = quotasFromData(data); quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); - quotaInfo.setQuotas(quotas); + quotaInfo.setQuotas(conf, quotas); } catch (IOException e) { LOG.error("Unable to parse {} '{}' quotas", type, key, e); globalQuotas.remove(key); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java index 232471092c29..43dfab703b74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.quotas; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -32,7 +31,6 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class TimeBasedLimiter implements QuotaLimiter { - private static final Configuration conf = HBaseConfiguration.create(); private RateLimiter reqsLimiter = null; private RateLimiter reqSizeLimiter = null; private RateLimiter writeReqsLimiter = null; @@ -47,7 +45,7 @@ public class TimeBasedLimiter implements QuotaLimiter { private RateLimiter atomicWriteSizeLimiter = null; private RateLimiter reqHandlerUsageTimeLimiter = null; - private TimeBasedLimiter() { + private TimeBasedLimiter(Configuration conf) { if ( FixedIntervalRateLimiter.class.getName().equals( conf.getClass(RateLimiter.QUOTA_RATE_LIMITER_CONF_KEY, AverageIntervalRateLimiter.class) @@ -85,8 +83,8 @@ private TimeBasedLimiter() { } } - static QuotaLimiter fromThrottle(final Throttle throttle) { - TimeBasedLimiter limiter = new TimeBasedLimiter(); + static QuotaLimiter fromThrottle(Configuration conf, final Throttle throttle) { + TimeBasedLimiter limiter = new TimeBasedLimiter(conf); boolean isBypass = true; if (throttle.hasReqNum()) { setFromTimedQuota(limiter.reqsLimiter, throttle.getReqNum()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java index 877ad195c716..0704e869239b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -89,8 +90,8 @@ public synchronized boolean hasBypassGlobals() { } @Override - public synchronized void setQuotas(final Quotas quotas) { - super.setQuotas(quotas); + public synchronized void setQuotas(Configuration conf, final Quotas quotas) { + super.setQuotas(conf, quotas); bypassGlobals = quotas.getBypassGlobals(); } @@ -98,30 +99,30 @@ public synchronized void setQuotas(final Quotas quotas) { * Add the quota information of the specified table. (This operation is part of the QuotaState * setup) */ - public synchronized void setQuotas(final TableName table, Quotas quotas) { - tableLimiters = setLimiter(tableLimiters, table, quotas); + public synchronized void setQuotas(Configuration conf, final TableName table, Quotas quotas) { + tableLimiters = setLimiter(conf, tableLimiters, table, quotas); } /** * Add the quota information of the specified namespace. (This operation is part of the QuotaState * setup) */ - public void setQuotas(final String namespace, Quotas quotas) { - namespaceLimiters = setLimiter(namespaceLimiters, namespace, quotas); + public void setQuotas(Configuration conf, final String namespace, Quotas quotas) { + namespaceLimiters = setLimiter(conf, namespaceLimiters, namespace, quotas); } public boolean hasTableLimiters() { return tableLimiters != null && !tableLimiters.isEmpty(); } - private Map setLimiter(Map limiters, final K key, - final Quotas quotas) { + private Map setLimiter(Configuration conf, Map limiters, + final K key, final Quotas quotas) { if (limiters == null) { limiters = new HashMap<>(); } QuotaLimiter limiter = - quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(quotas.getThrottle()) : null; + quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(conf, quotas.getThrottle()) : null; if (limiter != null && !limiter.isBypass()) { limiters.put(key, limiter); } else { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java index 4a638d965b38..e614e71b3350 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java @@ -43,6 +43,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ MediumTests.class, CoprocessorTests.class }) public class TestRegionCoprocessorQuotaUsage { @@ -52,6 +54,7 @@ public class TestRegionCoprocessorQuotaUsage { HBaseClassTestRule.forClass(TestRegionCoprocessorQuotaUsage.class); private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Logger LOG = LoggerFactory.getLogger(TestRegionCoprocessorQuotaUsage.class); private static TableName TABLE_NAME = TableName.valueOf("TestRegionCoprocessorQuotaUsage"); private static byte[] CF = Bytes.toBytes("CF"); private static byte[] CQ = Bytes.toBytes("CQ"); @@ -66,11 +69,14 @@ public void preGetOp(ObserverContext c, Get get, // For the purposes of this test, we only need to catch a throttle happening once, then // let future requests pass through so we don't make this test take any longer than necessary + LOG.info("Intercepting GetOp"); if (!THROTTLING_OCCURRED.get()) { try { c.getEnvironment().checkBatchQuota(c.getEnvironment().getRegion(), OperationQuota.OperationType.GET); + LOG.info("Request was not throttled"); } catch (RpcThrottlingException e) { + LOG.info("Intercepting was throttled"); THROTTLING_OCCURRED.set(true); throw e; } @@ -91,9 +97,8 @@ public Optional getRegionObserver() { public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setBoolean("hbase.quota.enabled", true); - conf.setInt("hbase.quota.default.user.machine.read.num", 2); + conf.setInt("hbase.quota.default.user.machine.read.num", 1); conf.set("hbase.quota.rate.limiter", "org.apache.hadoop.hbase.quotas.FixedIntervalRateLimiter"); - conf.set("hbase.quota.rate.limiter.refill.interval.ms", "300000"); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MyCoprocessor.class.getName()); UTIL.startMiniCluster(3); byte[][] splitKeys = new byte[8][]; @@ -116,6 +121,9 @@ public void testGet() throws InterruptedException, ExecutionException, IOExcepti // Hit the table 5 times which ought to be enough to make a throttle happen for (int i = 0; i < 5; i++) { TABLE.get(new Get(Bytes.toBytes("000"))); + if (THROTTLING_OCCURRED.get()) { + break; + } } assertTrue("Throttling did not happen as expected", THROTTLING_OCCURRED.get()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java index c22a03f8db00..2b9200ab6465 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -41,6 +42,7 @@ public class TestDefaultOperationQuota { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestDefaultOperationQuota.class); + private static final Configuration conf = HBaseConfiguration.create(); private static final int DEFAULT_REQUESTS_PER_SECOND = 1000; private static ManualEnvironmentEdge envEdge = new ManualEnvironmentEdge(); static { @@ -150,7 +152,7 @@ public void testLargeBatchSaturatesReadNumLimit() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -172,7 +174,7 @@ public void testLargeBatchSaturatesReadWriteLimit() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -194,7 +196,7 @@ public void testTooLargeReadBatchIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -216,7 +218,7 @@ public void testTooLargeWriteBatchIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -238,7 +240,7 @@ public void testTooLargeWriteSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -261,7 +263,7 @@ public void testTooLargeReadSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), (int) blockSize, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -284,7 +286,7 @@ public void testTooLargeRequestSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReqSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), (int) blockSize, DEFAULT_REQUESTS_PER_SECOND, limiter); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java index 2c33b265771a..8f8ac4991ca6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -23,7 +23,9 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.ClassRule; @@ -43,6 +45,8 @@ public class TestQuotaCache2 { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestQuotaCache2.class); + private static final Configuration conf = HBaseConfiguration.create(); + @Test public void testPreserveLimiterAvailability() throws Exception { // establish old cache with a limiter for 100 read bytes per second @@ -53,7 +57,7 @@ public void testPreserveLimiterAvailability() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); oldState.setGlobalLimiter(limiter1); // consume one byte from the limiter, so 99 will be left @@ -67,7 +71,7 @@ public void testPreserveLimiterAvailability() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); newState.setGlobalLimiter(limiter2); // update new cache from old cache @@ -89,7 +93,7 @@ public void testClobberLimiterLimit() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); oldState.setGlobalLimiter(limiter1); // establish new cache, also with a limiter for 100 read bytes per second @@ -100,7 +104,7 @@ public void testClobberLimiterLimit() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(50).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); newState.setGlobalLimiter(limiter2); // update new cache from old cache diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java index ff4b6bc9949b..b45f78b07653 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -22,7 +22,9 @@ import static org.junit.Assert.fail; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -48,6 +50,8 @@ public class TestQuotaState { @Rule public TestName name = new TestName(); + private static final Configuration conf = HBaseConfiguration.create(); + @Test public void testQuotaStateBypass() { QuotaState quotaInfo = new QuotaState(); @@ -69,11 +73,11 @@ public void testSimpleQuotaStateOperation() { assertTrue(quotaInfo.isBypass()); // Set global quota - quotaInfo.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE)); + quotaInfo.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE)); assertFalse(quotaInfo.isBypass()); // Set table quota - quotaInfo.setQuotas(tableName, buildReqNumThrottle(NUM_TABLE_THROTTLE)); + quotaInfo.setQuotas(conf, tableName, buildReqNumThrottle(NUM_TABLE_THROTTLE)); assertFalse(quotaInfo.isBypass()); assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); assertThrottleException(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME), NUM_GLOBAL_THROTTLE); @@ -90,7 +94,7 @@ public void testQuotaStateUpdateGlobalThrottle() { // Add global throttle QuotaState otherQuotaState = new QuotaState(); - otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); + otherQuotaState.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -99,7 +103,7 @@ public void testQuotaStateUpdateGlobalThrottle() { // Update global Throttle otherQuotaState = new QuotaState(); - otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); + otherQuotaState.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -131,8 +135,8 @@ public void testQuotaStateUpdateTableThrottle() { // Add A B table limiters UserQuotaState otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); - otherQuotaState.setQuotas(tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); + otherQuotaState.setQuotas(conf, tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + otherQuotaState.setQuotas(conf, tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -143,8 +147,8 @@ public void testQuotaStateUpdateTableThrottle() { // Add C, Remove B, Update A table limiters otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); - otherQuotaState.setQuotas(tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); + otherQuotaState.setQuotas(conf, tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); + otherQuotaState.setQuotas(conf, tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -173,7 +177,7 @@ public void testTableThrottleWithBatch() { // Add A table limiters UserQuotaState otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + otherQuotaState.setQuotas(conf, TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); From 4cad08da0b9b07427c0c6388d890b31a3b3cb1c7 Mon Sep 17 00:00:00 2001 From: gvprathyusha6 <70918688+gvprathyusha6@users.noreply.github.com> Date: Tue, 21 Oct 2025 02:12:20 +0530 Subject: [PATCH 088/336] HBASE-28564 Refactor direct interactions of Reference file creations to SFT interface (#5939) (#7382) Signed-off-by: Andrew Purtell Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java --- .../org/apache/hadoop/hbase/io/HFileLink.java | 2 +- .../org/apache/hadoop/hbase/io/Reference.java | 2 +- .../hadoop/hbase/io/hfile/CacheConfig.java | 1 - .../MergeTableRegionsProcedure.java | 2 +- .../assignment/SplitTableRegionProcedure.java | 37 ++-- .../hbase/master/janitor/CatalogJanitor.java | 15 +- .../hadoop/hbase/mob/CachedMobFile.java | 5 +- .../hbase/mob/ExpiredMobFileCleaner.java | 8 +- .../org/apache/hadoop/hbase/mob/MobFile.java | 7 +- .../apache/hadoop/hbase/mob/MobFileCache.java | 11 +- .../hadoop/hbase/mob/MobFileCleanerChore.java | 2 +- .../hadoop/hbase/mob/MobFileCleanupUtil.java | 24 +-- .../org/apache/hadoop/hbase/mob/MobUtils.java | 15 +- .../regionserver/DataTieringManager.java | 2 +- .../hadoop/hbase/regionserver/HMobStore.java | 7 +- .../hadoop/hbase/regionserver/HRegion.java | 25 ++- .../hbase/regionserver/HRegionFileSystem.java | 84 ++------ .../hadoop/hbase/regionserver/HStore.java | 10 +- .../hadoop/hbase/regionserver/HStoreFile.java | 5 +- .../hbase/regionserver/StoreEngine.java | 7 +- .../hbase/regionserver/StoreFileInfo.java | 63 +++--- .../DefaultStoreFileTracker.java | 50 ++++- .../FileBasedStoreFileTracker.java | 19 +- .../storefiletracker/StoreFileTracker.java | 25 +++ .../StoreFileTrackerBase.java | 128 ++++++++++++ .../StoreFileTrackerFactory.java | 8 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 59 +++--- .../hbase/snapshot/SnapshotManifest.java | 20 +- .../hbase/snapshot/SnapshotManifestV1.java | 43 +++- .../hbase/util/ServerRegionReplicaUtil.java | 9 +- .../compaction/MajorCompactionRequest.java | 15 +- .../hbase-webapps/regionserver/storeFile.jsp | 2 +- .../client/TestTableSnapshotScanner.java | 18 +- .../hbase/io/TestHalfStoreFileReader.java | 38 +++- .../hbase/io/hfile/TestBytesReadFromFs.java | 2 +- .../hadoop/hbase/io/hfile/TestPrefetch.java | 24 ++- .../io/hfile/TestPrefetchWithBucketCache.java | 13 +- .../master/janitor/TestCatalogJanitor.java | 48 ++++- .../hadoop/hbase/mob/TestCachedMobFile.java | 23 ++- .../hbase/mob/TestExpiredMobFileCleaner.java | 7 + .../apache/hadoop/hbase/mob/TestMobFile.java | 11 +- .../hadoop/hbase/mob/TestMobFileCache.java | 63 +++--- .../hbase/mob/TestMobStoreCompaction.java | 13 +- ...bstractTestDateTieredCompactionPolicy.java | 7 +- .../regionserver/DataBlockEncodingTool.java | 3 +- .../EncodedSeekPerformanceTest.java | 10 +- .../hbase/regionserver/MockHStoreFile.java | 19 +- .../TestCacheOnWriteInSchema.java | 3 +- .../TestCompactionArchiveIOException.java | 6 +- .../regionserver/TestCompactionPolicy.java | 7 +- .../regionserver/TestCompoundBloomFilter.java | 3 +- .../TestCustomCellDataTieringManager.java | 7 +- .../TestCustomCellTieredCompactionPolicy.java | 10 +- .../regionserver/TestDataTieringManager.java | 8 +- .../TestDirectStoreSplitsMerges.java | 45 ++-- .../regionserver/TestFSErrorsExposed.java | 10 +- .../hbase/regionserver/TestHRegion.java | 12 +- .../regionserver/TestHRegionFileSystem.java | 17 +- .../hadoop/hbase/regionserver/TestHStore.java | 8 +- .../hbase/regionserver/TestHStoreFile.java | 194 ++++++++++++------ .../TestMergesSplitsAddToTracker.java | 35 +++- .../TestRegionMergeTransactionOnCluster.java | 14 +- .../regionserver/TestReversibleScanners.java | 25 ++- .../TestRowPrefixBloomFilter.java | 6 +- .../TestSplitTransactionOnCluster.java | 13 +- .../hbase/regionserver/TestStoreFileInfo.java | 35 ++-- .../TestStoreFileRefresherChore.java | 32 +-- ...estStoreFileScannerWithTagCompression.java | 2 +- .../regionserver/TestStoreScannerClosure.java | 10 +- .../TestStripeStoreFileManager.java | 5 +- .../FailingStoreFileTrackerForTest.java | 42 ++++ .../StoreFileTrackerForTest.java | 7 + .../snapshot/TestSnapshotStoreFileSize.java | 9 +- .../TestMajorCompactionRequest.java | 19 +- .../TestMajorCompactionTTLRequest.java | 4 + 75 files changed, 1133 insertions(+), 466 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FailingStoreFileTrackerForTest.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index a036a90d7cf7..dc7ac7338acc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -463,7 +463,7 @@ public static String createFromHFileLink(final Configuration conf, final FileSys * Create the back reference name */ // package-private for testing - static String createBackReferenceName(final String tableNameStr, final String regionName) { + public static String createBackReferenceName(final String tableNameStr, final String regionName) { return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '='); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java index 337fde60cf7d..22d3c9ce2c0b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java @@ -195,7 +195,7 @@ public static Reference convert(final FSProtos.Reference r) { * delimiter, pb reads to EOF which may not be what you want). * @return This instance serialized as a delimited protobuf w/ a magic pb prefix. */ - byte[] toByteArray() throws IOException { + public byte[] toByteArray() throws IOException { return ProtobufUtil.prependPBMagic(convert().toByteArray()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index ae196340db61..e65bfc34073f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -162,7 +162,6 @@ public class CacheConfig implements PropagatingConfigurationObserver { private final ByteBuffAllocator byteBuffAllocator; - /** * Create a cache configuration using the specified configuration object and defaults for family * level settings. Only use if no column family context. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 8203a3458372..4c42229e3e6f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -644,7 +644,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg // to read the hfiles. storeFileInfo.setConf(storeConfiguration); Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, - new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker); mergedFiles.add(refFile); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 3250680d57bb..3e43079003fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -701,8 +701,9 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en // table dir. In case of failure, the proc would go through this again, already existing // region dirs and split files would just be ignored, new split files should get created. int nbFiles = 0; - final Map> files = - new HashMap>(htd.getColumnFamilyCount()); + final Map, StoreFileTracker>> files = + new HashMap, StoreFileTracker>>( + htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); StoreFileTracker tracker = @@ -725,7 +726,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en } if (filteredSfis == null) { filteredSfis = new ArrayList(sfis.size()); - files.put(family, filteredSfis); + files.put(family, new Pair(filteredSfis, tracker)); } filteredSfis.add(sfi); nbFiles++; @@ -748,10 +749,12 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en final List>> futures = new ArrayList>>(nbFiles); // Split each store file. - for (Map.Entry> e : files.entrySet()) { + for (Map.Entry, StoreFileTracker>> e : files + .entrySet()) { byte[] familyName = Bytes.toBytes(e.getKey()); final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName); - final Collection storeFiles = e.getValue(); + Pair, StoreFileTracker> storeFilesAndTracker = e.getValue(); + final Collection storeFiles = storeFilesAndTracker.getFirst(); if (storeFiles != null && storeFiles.size() > 0) { final Configuration storeConfiguration = StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); @@ -762,8 +765,9 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en // is running in a regionserver's Store context, or we might not be able // to read the hfiles. storeFileInfo.setConf(storeConfiguration); - StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName, - new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + StoreFileSplitter sfs = + new StoreFileSplitter(regionFs, storeFilesAndTracker.getSecond(), familyName, + new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); futures.add(threadPool.submit(sfs)); } } @@ -829,8 +833,8 @@ private void assertSplitResultFilesCount(final FileSystem fs, } } - private Pair splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) - throws IOException { + private Pair splitStoreFile(HRegionFileSystem regionFs, StoreFileTracker tracker, + byte[] family, HStoreFile sf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); @@ -838,10 +842,10 @@ private Pair splitStoreFile(HRegionFileSystem regionFs, byte[] famil final byte[] splitRow = getSplitRow(); final String familyName = Bytes.toString(family); - final Path path_first = - regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy); - final Path path_second = - regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy); + final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, + false, splitPolicy, tracker); + final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, + true, splitPolicy, tracker); if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); @@ -857,6 +861,7 @@ private class StoreFileSplitter implements Callable> { private final HRegionFileSystem regionFs; private final byte[] family; private final HStoreFile sf; + private final StoreFileTracker tracker; /** * Constructor that takes what it needs to split @@ -864,15 +869,17 @@ private class StoreFileSplitter implements Callable> { * @param family Family that contains the store file * @param sf which file */ - public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) { + public StoreFileSplitter(HRegionFileSystem regionFs, StoreFileTracker tracker, byte[] family, + HStoreFile sf) { this.regionFs = regionFs; this.sf = sf; this.family = family; + this.tracker = tracker; } @Override public Pair call() throws IOException { - return splitStoreFile(regionFs, family, sf); + return splitStoreFile(regionFs, tracker, family, sf); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java index 8b482b3ae019..71d1d1ced63f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; @@ -49,6 +51,8 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -421,7 +425,16 @@ private static Pair checkRegionReferences(MasterServices servi try { HRegionFileSystem regionFs = HRegionFileSystem .openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true); - boolean references = regionFs.hasReferences(tableDescriptor); + ColumnFamilyDescriptor[] families = tableDescriptor.getColumnFamilies(); + boolean references = false; + for (ColumnFamilyDescriptor cfd : families) { + StoreFileTracker sft = StoreFileTrackerFactory.create(services.getConfiguration(), + tableDescriptor, ColumnFamilyDescriptorBuilder.of(cfd.getNameAsString()), regionFs); + references = references || sft.hasReferences(); + if (references) { + break; + } + } return new Pair<>(Boolean.TRUE, references); } catch (IOException e) { LOG.error("Error trying to determine if region {} has references, assuming it does", diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java index 1c1145a2f482..cdf941878119 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.yetus.audience.InterfaceAudience; /** @@ -41,10 +42,10 @@ public CachedMobFile(HStoreFile sf) { } public static CachedMobFile create(FileSystem fs, Path path, Configuration conf, - CacheConfig cacheConf) throws IOException { + CacheConfig cacheConf, StoreFileTracker sft) throws IOException { // XXX: primaryReplica is only used for constructing the key of block cache so it is not a // critical problem if we pass the wrong value, so here we always pass true. Need to fix later. - HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); + HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft); return new CachedMobFile(sf); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java index 3c02d483c0b6..9ecda5ec8cb8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java @@ -57,17 +57,17 @@ public class ExpiredMobFileCleaner extends Configured implements Tool { * @param tableName The current table name. * @param family The current family. */ - public void cleanExpiredMobFiles(String tableName, ColumnFamilyDescriptor family) + public void cleanExpiredMobFiles(TableDescriptor htd, ColumnFamilyDescriptor family) throws IOException { Configuration conf = getConf(); - TableName tn = TableName.valueOf(tableName); + String tableName = htd.getTableName().getNameAsString(); FileSystem fs = FileSystem.get(conf); LOG.info("Cleaning the expired MOB files of " + family.getNameAsString() + " in " + tableName); // disable the block cache. Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); - MobUtils.cleanExpiredMobFiles(fs, conf, tn, family, cacheConfig, + MobUtils.cleanExpiredMobFiles(fs, conf, htd, family, cacheConfig, EnvironmentEdgeManager.currentTime()); } @@ -107,7 +107,7 @@ public int run(String[] args) throws Exception { throw new IOException( "The minVersions of the column family is not 0, could not be handled by this cleaner"); } - cleanExpiredMobFiles(tableName, family); + cleanExpiredMobFiles(htd, family); return 0; } finally { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java index 3293208771ac..de7f61032ed5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.yetus.audience.InterfaceAudience; /** @@ -133,11 +134,11 @@ public void close() throws IOException { * @param cacheConf The CacheConfig. * @return An instance of the MobFile. */ - public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf) - throws IOException { + public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf, + StoreFileTracker sft) throws IOException { // XXX: primaryReplica is only used for constructing the key of block cache so it is not a // critical problem if we pass the wrong value, so here we always pass true. Need to fix later. - HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); + HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft); return new MobFile(sf); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java index b353b53ffb71..45ec006f97f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java @@ -33,6 +33,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.IdLock; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -198,9 +201,11 @@ public void evictFile(String fileName) { * @param cacheConf The current MobCacheConfig * @return A opened mob file. */ - public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException { + public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf, + StoreContext storeContext) throws IOException { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); if (!isCacheEnabled) { - MobFile mobFile = MobFile.create(fs, path, conf, cacheConf); + MobFile mobFile = MobFile.create(fs, path, conf, cacheConf, sft); mobFile.open(); return mobFile; } else { @@ -214,7 +219,7 @@ public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws if (map.size() > mobFileMaxCacheSize) { evict(); } - cached = CachedMobFile.create(fs, path, conf, cacheConf); + cached = CachedMobFile.create(fs, path, conf, cacheConf, sft); cached.open(); map.put(fileName, cached); miss.increment(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index c4bada278df6..9ce20e7c650e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -92,7 +92,7 @@ protected void chore() { for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) { try { - cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd); + cleaner.cleanExpiredMobFiles(htd, hcd); } catch (IOException e) { LOG.error("Failed to clean the expired mob files table={} family={}", htd.getTableName().getNameAsString(), hcd.getNameAsString(), e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java index 049192624ef3..a1b2d4a792e8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java @@ -35,7 +35,11 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -90,7 +94,10 @@ public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Set allActiveMobFileName = new HashSet(); for (Path regionPath : regionDirs) { regionNames.add(regionPath.getName()); + HRegionFileSystem regionFS = + HRegionFileSystem.create(conf, fs, tableDir, MobUtils.getMobRegionInfo(table)); for (ColumnFamilyDescriptor hcd : list) { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false); String family = hcd.getNameAsString(); Path storePath = new Path(regionPath, family); boolean succeed = false; @@ -102,26 +109,19 @@ public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, + " execution, aborting MOB file cleaner chore.", storePath); throw new IOException(errMsg); } - RemoteIterator rit = fs.listLocatedStatus(storePath); - List storeFiles = new ArrayList(); - // Load list of store files first - while (rit.hasNext()) { - Path p = rit.next().getPath(); - if (fs.isFile(p)) { - storeFiles.add(p); - } - } - LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); + List storeFileInfos = sft.load(); + LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath); Path currentPath = null; try { - for (Path pp : storeFiles) { + for (StoreFileInfo storeFileInfo : storeFileInfos) { + Path pp = storeFileInfo.getPath(); currentPath = pp; LOG.trace("Store file: {}", pp); HStoreFile sf = null; byte[] mobRefData = null; byte[] bulkloadMarkerData = null; try { - sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); + sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED); sf.initReader(); mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index b6b8be9d1791..c8e6fac9ceda 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@ -59,9 +59,12 @@ import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -266,7 +269,7 @@ public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) { * @param cacheConfig The cacheConfig that disables the block cache. * @param current The current time. */ - public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, + public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableDescriptor htd, ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current) throws IOException { long timeToLive = columnDescriptor.getTimeToLive(); @@ -287,7 +290,11 @@ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, Table LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!"); FileStatus[] stats = null; + TableName tableName = htd.getTableName(); Path mobTableDir = CommonFSUtils.getTableDir(getMobHome(conf), tableName); + HRegionFileSystem regionFS = + HRegionFileSystem.create(conf, fs, mobTableDir, getMobRegionInfo(tableName)); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, columnDescriptor, regionFS); Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString()); try { stats = fs.listStatus(path); @@ -318,7 +325,7 @@ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, Table LOG.debug("{} is an expired file", fileName); } filesToClean - .add(new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true)); + .add(new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true, sft)); if ( filesToClean.size() >= conf.getInt(MOB_CLEANER_BATCH_SIZE_UPPER_BOUND, DEFAULT_MOB_CLEANER_BATCH_SIZE_UPPER_BOUND) @@ -387,6 +394,10 @@ public static Path getMobTableDir(Path rootDir, TableName tableName) { return CommonFSUtils.getTableDir(getMobHome(rootDir), tableName); } + public static Path getMobTableDir(Configuration conf, TableName tableName) { + return getMobTableDir(new Path(conf.get(HConstants.HBASE_DIR)), tableName); + } + /** * Gets the region dir of the mob files. It's * {HBASE_DIR}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 8443827ccaa1..2a5e2a5aa39d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -254,7 +254,7 @@ private HStore getHStore(Path hFilePath) throws DataTieringException { private HStoreFile getHStoreFile(Path hFilePath) throws DataTieringException { HStore hStore = getHStore(hFilePath); for (HStoreFile file : hStore.getStorefiles()) { - if (file.getPath().equals(hFilePath)) { + if (file.getPath().toUri().getPath().toString().equals(hFilePath.toString())) { return file; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index d4b24de33cc3..468f478dbc48 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -56,6 +56,8 @@ import org.apache.hadoop.hbase.mob.MobFileName; import org.apache.hadoop.hbase.mob.MobStoreEngine; import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.IdLock; import org.apache.yetus.audience.InterfaceAudience; @@ -280,8 +282,9 @@ public void commitFile(final Path sourceFile, Path targetPath) throws IOExceptio private void validateMobFile(Path path) throws IOException { HStoreFile storeFile = null; try { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, getStoreContext()); storeFile = new HStoreFile(getFileSystem(), path, conf, getCacheConfig(), BloomType.NONE, - isPrimaryReplicaStore()); + isPrimaryReplicaStore(), sft); storeFile.initReader(); } catch (IOException e) { LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e); @@ -405,7 +408,7 @@ private MobCell readCell(List locations, String fileName, Cell search, MobFile file = null; Path path = new Path(location, fileName); try { - file = mobFileCache.openFile(fs, path, getCacheConfig()); + file = mobFileCache.openFile(fs, path, getCacheConfig(), this.getStoreContext()); return readPt != -1 ? file.readCell(search, cacheMobBlocks, readPt) : file.readCell(search, cacheMobBlocks); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index e550e8ba8df5..f75e8f5ac5e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -156,6 +156,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector; @@ -1314,7 +1316,9 @@ public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration if (StoreFileInfo.isReference(p) || HFileLink.isHFileLink(p)) { // Only construct StoreFileInfo object if its not a hfile, save obj // creation - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status); + StoreFileTracker sft = + StoreFileTrackerFactory.create(conf, tableDescriptor, family, regionFs); + StoreFileInfo storeFileInfo = sft.getStoreFileInfo(status, status.getPath(), false); hdfsBlocksDistribution.add(storeFileInfo.computeHDFSBlocksDistribution(fs)); } else if (StoreFileInfo.isHFile(p)) { // If its a HFile, then lets just add to the block distribution @@ -5307,9 +5311,12 @@ long replayRecoveredEditsIfAny(Map maxSeqIdInStores, // column family. Have to fake out file type too by casting our recovered.edits as // storefiles String fakeFamilyName = WALSplitUtil.getRegionDirRecoveredEditsDir(regionWALDir).getName(); + StoreContext storeContext = + StoreContext.getBuilder().withRegionFileSystem(getRegionFileSystem()).build(); + StoreFileTracker sft = StoreFileTrackerFactory.create(this.conf, true, storeContext); Set fakeStoreFiles = new HashSet<>(files.size()); for (Path file : files) { - fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true)); + fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true, sft)); } getRegionWALFileSystem().archiveRecoveredEdits(fakeFamilyName, fakeStoreFiles); } else { @@ -6295,17 +6302,15 @@ void replayWALBulkLoadEventMarker(WALProtos.BulkLoadDescriptor bulkLoadEvent) th continue; } - List storeFiles = storeDescriptor.getStoreFileList(); - for (String storeFile : storeFiles) { - StoreFileInfo storeFileInfo = null; + StoreContext storeContext = store.getStoreContext(); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); + + List storeFiles = sft.load(); + for (StoreFileInfo storeFileInfo : storeFiles) { try { - storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile); store.bulkLoadHFile(storeFileInfo); } catch (FileNotFoundException ex) { - LOG.warn(getRegionInfo().getEncodedName() + " : " - + ((storeFileInfo != null) - ? storeFileInfo.toString() - : (new Path(Bytes.toString(family), storeFile)).toString()) + LOG.warn(getRegionInfo().getEncodedName() + " : " + storeFileInfo.toString() + " doesn't exist any more. Skip loading the file"); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index c77f4d4aefde..b80599fd61a3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -255,51 +255,6 @@ public String getStoragePolicyName(String familyName) { return null; } - /** - * Returns the store files available for the family. This methods performs the filtering based on - * the valid store files. - * @param familyName Column Family Name - * @return a set of {@link StoreFileInfo} for the specified family. - */ - public List getStoreFiles(final String familyName) throws IOException { - return getStoreFiles(familyName, true); - } - - /** - * Returns the store files available for the family. This methods performs the filtering based on - * the valid store files. - * @param familyName Column Family Name - * @return a set of {@link StoreFileInfo} for the specified family. - */ - public List getStoreFiles(final String familyName, final boolean validate) - throws IOException { - Path familyDir = getStoreDir(familyName); - FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir); - if (files == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No StoreFiles for: " + familyDir); - } - return null; - } - - ArrayList storeFiles = new ArrayList<>(files.length); - for (FileStatus status : files) { - if (validate && !StoreFileInfo.isValid(status)) { - // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to - // true, refer HBASE-23740 - if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) { - LOG.warn("Invalid StoreFile: {}", status.getPath()); - } - continue; - } - StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, - regionInfoForFs, familyName, status.getPath()); - storeFiles.add(info); - - } - return storeFiles; - } - /** * Returns the store files' LocatedFileStatus which available for the family. This methods * performs the filtering based on the valid store files. @@ -350,11 +305,11 @@ Path getStoreFilePath(final String familyName, final String fileName) { * @param fileName File Name * @return The {@link StoreFileInfo} for the specified family/file */ - StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) - throws IOException { + StoreFileInfo getStoreFileInfo(final String familyName, final String fileName, + final StoreFileTracker tracker) throws IOException { Path familyDir = getStoreDir(familyName); return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs, - familyName, new Path(familyDir, fileName)); + familyName, new Path(familyDir, fileName), tracker); } /** @@ -379,20 +334,6 @@ public boolean hasReferences(final String familyName) throws IOException { return false; } - /** - * Check whether region has Reference file - * @param htd table desciptor of the region - * @return true if region has reference file - */ - public boolean hasReferences(final TableDescriptor htd) throws IOException { - for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) { - if (hasReferences(family.getNameAsString())) { - return true; - } - } - return false; - } - /** Returns the set of families present on disk n */ public Collection getFamilies() throws IOException { FileStatus[] fds = @@ -628,7 +569,7 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); - infos.add(new StoreFileInfo(conf, fs, file, true)); + infos.add(trackerMap.get(familyName).getStoreFileInfo(file, true)); } for (Map.Entry entry : trackerMap.entrySet()) { entry.getValue().add(fileInfoMap.get(entry.getKey())); @@ -672,7 +613,7 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * @return Path to created reference. */ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, - boolean top, RegionSplitPolicy splitPolicy) throws IOException { + boolean top, RegionSplitPolicy splitPolicy, StoreFileTracker tracker) throws IOException { Path splitDir = new Path(getSplitsDir(hri), familyName); // Add the referred-to regions name as a dot separated suffix. // See REF_NAME_REGEX regex above. The referred-to regions name is @@ -758,7 +699,8 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte // A reference to the bottom half of the hsf store file. Reference r = top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow); - return r.write(fs, p); + tracker.createReference(r, p); + return p; } // =========================================================================== @@ -799,8 +741,8 @@ static boolean mkdirs(FileSystem fs, Configuration conf, Path dir) throws IOExce * @return Path to created reference. * @throws IOException if the merge write fails. */ - public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f) - throws IOException { + public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f, + StoreFileTracker tracker) throws IOException { Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName); // A whole reference to the store file. Reference r = Reference.createTopReference(mergingRegion.getStartKey()); @@ -812,7 +754,8 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi // Write reference with same file id only with the other region name as // suffix and into the new region location (under same family). Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName); - return r.write(fs, p); + tracker.createReference(r, p); + return p; } /** @@ -1222,4 +1165,9 @@ private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSl } Thread.sleep((long) baseSleepBeforeRetries * sleepMultiplier); } + + public static HRegionFileSystem create(final Configuration conf, final FileSystem fs, + final Path tableDir, final RegionInfo regionInfo) throws IOException { + return new HRegionFileSystem(conf, fs, tableDir, regionInfo); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 1f37d08f7299..d63633e5311d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -88,6 +88,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.EncryptionUtil; @@ -431,7 +433,7 @@ public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) { return ttl; } - StoreContext getStoreContext() { + public StoreContext getStoreContext() { return storeContext; } @@ -1399,8 +1401,9 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick compactionOutputs.remove(sf.getPath().getName()); } for (String compactionOutput : compactionOutputs) { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); StoreFileInfo storeFileInfo = - getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput); + getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); outputStoreFiles.add(storeFile); } @@ -2043,8 +2046,9 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) List storeFiles = new ArrayList<>(fileNames.size()); for (String file : fileNames) { // open the file as a store file (hfile link, etc) + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); StoreFileInfo storeFileInfo = - getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file); + getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); storeFiles.add(storeFile); HStore.this.storeSize.addAndGet(storeFile.getReader().length()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java index 14627ebc9389..d52abdba1fc3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; @@ -226,8 +227,8 @@ public long getMaxMemStoreTS() { * @param primaryReplica true if this is a store file for primary replica, otherwise false. */ public HStoreFile(FileSystem fs, Path p, Configuration conf, CacheConfig cacheConf, - BloomType cfBloomType, boolean primaryReplica) throws IOException { - this(new StoreFileInfo(conf, fs, p, primaryReplica), cfBloomType, cacheConf); + BloomType cfBloomType, boolean primaryReplica, StoreFileTracker sft) throws IOException { + this(sft.getStoreFileInfo(p, primaryReplica), cfBloomType, cacheConf); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index a696032d59f4..20e8328f0efa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -220,8 +220,7 @@ public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws I } public HStoreFile createStoreFileAndReader(Path p) throws IOException { - StoreFileInfo info = new StoreFileInfo(conf, ctx.getRegionFileSystem().getFileSystem(), p, - ctx.isPrimaryReplicaStore()); + StoreFileInfo info = storeFileTracker.getStoreFileInfo(p, ctx.isPrimaryReplicaStore()); return createStoreFileAndReader(info); } @@ -373,8 +372,8 @@ public void refreshStoreFiles() throws IOException { public void refreshStoreFiles(Collection newFiles) throws IOException { List storeFiles = new ArrayList<>(newFiles.size()); for (String file : newFiles) { - storeFiles - .add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), file)); + storeFiles.add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), + file, storeFileTracker)); } refreshStoreFilesInternal(storeFiles); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index 290f8b799ca9..6d866613dd16 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -35,10 +35,12 @@ import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.hadoop.hbase.io.hfile.InvalidHFileException; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; @@ -111,20 +113,9 @@ public class StoreFileInfo implements Configurable { // done. private final AtomicInteger refCount = new AtomicInteger(0); - /** - * Create a Store File Info - * @param conf the {@link Configuration} to use - * @param fs The current file system to use. - * @param initialPath The {@link Path} of the file - * @param primaryReplica true if this is a store file for primary replica, otherwise false. - */ - public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath, - final boolean primaryReplica) throws IOException { - this(conf, fs, null, initialPath, primaryReplica); - } - private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, - final Path initialPath, final boolean primaryReplica) throws IOException { + final Path initialPath, final boolean primaryReplica, final StoreFileTracker sft) + throws IOException { assert fs != null; assert initialPath != null; assert conf != null; @@ -142,7 +133,7 @@ private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileS this.link = HFileLink.buildFromHFileLinkPattern(conf, p); LOG.trace("{} is a link", p); } else if (isReference(p)) { - this.reference = Reference.read(fs, p); + this.reference = sft.readReference(p); Path referencePath = getReferredToFile(p); if (HFileLink.isHFileLink(referencePath)) { // HFileLink Reference @@ -169,17 +160,6 @@ private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileS } } - /** - * Create a Store File Info - * @param conf the {@link Configuration} to use - * @param fs The current file system to use. - * @param fileStatus The {@link FileStatus} of the file - */ - public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus) - throws IOException { - this(conf, fs, fileStatus, fileStatus.getPath(), true); - } - /** * Create a Store File Info from an HFileLink * @param conf The {@link Configuration} to use @@ -224,6 +204,29 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD); } + /** + * Create a Store File Info from an HFileLink and a Reference + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + * @param reference The reference instance + * @param link The link instance + */ + public StoreFileInfo(final Configuration conf, final FileSystem fs, final long createdTimestamp, + final Path initialPath, final long size, final Reference reference, final HFileLink link, + final boolean primaryReplica) { + this.fs = fs; + this.conf = conf; + this.primaryReplica = primaryReplica; + this.initialPath = initialPath; + this.createdTimestamp = createdTimestamp; + this.size = size; + this.reference = reference; + this.link = link; + this.noReadahead = + this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD); + } + @Override public Configuration getConf() { return conf; @@ -769,4 +772,14 @@ int decreaseRefCount() { return this.refCount.decrementAndGet(); } + public static StoreFileInfo createStoreFileInfoForHFile(final Configuration conf, + final FileSystem fs, final Path initialPath, final boolean primaryReplica) throws IOException { + if (HFileLink.isHFileLink(initialPath) || isReference(initialPath)) { + throw new InvalidHFileException("Path " + initialPath + " is a Hfile link or a Regerence"); + } + StoreFileInfo storeFileInfo = + new StoreFileInfo(conf, fs, null, initialPath, primaryReplica, null); + return storeFileInfo; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index 128537f10afe..035e6c76f85a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -18,13 +18,21 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The default implementation for store file tracker, where we do not persist the store file list, @@ -37,6 +45,8 @@ public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, Sto super(conf, isPrimaryReplica, ctx); } + private static final Logger LOG = LoggerFactory.getLogger(DefaultStoreFileTracker.class); + @Override public boolean requireWritingToTmpDirFirst() { return true; @@ -55,12 +65,48 @@ protected void doAddCompactionResults(Collection compactedFiles, @Override protected List doLoadStoreFiles(boolean readOnly) throws IOException { - List files = - ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + List files = getStoreFiles(ctx.getFamily().getNameAsString()); return files != null ? files : Collections.emptyList(); } @Override protected void doSetStoreFiles(Collection files) throws IOException { } + + /** + * Returns the store files available for the family. This methods performs the filtering based on + * the valid store files. + * @param familyName Column Family Name + * @return a set of {@link StoreFileInfo} for the specified family. + */ + public List getStoreFiles(final String familyName) throws IOException { + Path familyDir = ctx.getRegionFileSystem().getStoreDir(familyName); + FileStatus[] files = + CommonFSUtils.listStatus(ctx.getRegionFileSystem().getFileSystem(), familyDir); + if (files == null) { + if (LOG.isTraceEnabled()) { + LOG.trace("No StoreFiles for: " + familyDir); + } + return null; + } + + ArrayList storeFiles = new ArrayList<>(files.length); + for (FileStatus status : files) { + if (!StoreFileInfo.isValid(status)) { + // recovered.hfiles directory is expected inside CF path when + // hbase.wal.split.to.hfile to + // true, refer HBASE-23740 + if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) { + LOG.warn("Invalid StoreFile: {}", status.getPath()); + } + continue; + } + StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, + ctx.getRegionFileSystem().getFileSystem(), ctx.getRegionInfo(), + ctx.getRegionFileSystem().getRegionInfoForFS(), familyName, status.getPath(), this); + storeFiles.add(info); + + } + return storeFiles; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index d3dfe21521d7..b000d837d59b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -33,6 +33,8 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileEntry; import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; @@ -53,6 +55,7 @@ class FileBasedStoreFileTracker extends StoreFileTrackerBase { private final StoreFileListFile backedFile; private final Map storefiles = new HashMap<>(); + private static final Logger LOG = LoggerFactory.getLogger(FileBasedStoreFileTracker.class); public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { super(conf, isPrimaryReplica, ctx); @@ -69,6 +72,10 @@ public FileBasedStoreFileTracker(Configuration conf, boolean isPrimaryReplica, S @Override protected List doLoadStoreFiles(boolean readOnly) throws IOException { StoreFileList list = backedFile.load(readOnly); + if (LOG.isTraceEnabled()) { + LOG.trace("Loaded file list backed file, containing " + list.getStoreFileList().size() + + " store file entries"); + } if (list == null) { return Collections.emptyList(); } @@ -77,7 +84,7 @@ protected List doLoadStoreFiles(boolean readOnly) throws IOExcept for (StoreFileEntry entry : list.getStoreFileList()) { infos.add(ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, ctx.getRegionInfo(), ctx.getRegionFileSystem().getRegionInfoForFS(), ctx.getFamily().getNameAsString(), - new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()))); + new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()), this)); } // In general, for primary replica, the load method should only be called once when // initialization, so we do not need synchronized here. And for secondary replicas, though the @@ -115,6 +122,9 @@ protected void doAddNewStoreFiles(Collection newFiles) throws IOE builder.addStoreFile(toStoreFileEntry(info)); } backedFile.update(builder); + if (LOG.isTraceEnabled()) { + LOG.trace(newFiles.size() + " store files added to store file list file: " + newFiles); + } for (StoreFileInfo info : newFiles) { storefiles.put(info.getPath().getName(), info); } @@ -138,6 +148,10 @@ protected void doAddCompactionResults(Collection compactedFiles, builder.addStoreFile(toStoreFileEntry(info)); } backedFile.update(builder); + if (LOG.isTraceEnabled()) { + LOG.trace( + "replace compacted files: " + compactedFileNames + " with new store files: " + newFiles); + } for (String name : compactedFileNames) { storefiles.remove(name); } @@ -157,6 +171,9 @@ protected void doSetStoreFiles(Collection files) throws IOExcepti builder.addStoreFile(toStoreFileEntry(info)); } backedFile.update(builder); + if (LOG.isTraceEnabled()) { + LOG.trace("Set store files in store file list file: " + files); + } } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index b0024b73786a..12343b50dd37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -20,7 +20,10 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -94,4 +97,26 @@ void replace(Collection compactedFiles, Collection * does not allow broken store files under the actual data directory. */ boolean requireWritingToTmpDirFirst(); + + Reference createReference(Reference reference, Path path) throws IOException; + + /** + * Reads the reference file from the given path. + * @param path the {@link Path} to the reference file in the file system. + * @return a {@link Reference} that points at top/bottom half of a an hfile + */ + Reference readReference(Path path) throws IOException; + + /** + * Returns true if the specified family has reference files + * @return true if family contains reference files + */ + boolean hasReferences() throws IOException; + + StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath, + final boolean primaryReplica) throws IOException; + + StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica) + throws IOException; + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index 794a707062e5..5d0b5b4ae08d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -19,13 +19,22 @@ import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; +import java.io.BufferedInputStream; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Collection; import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -37,11 +46,14 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + /** * Base class for all store file tracker. *

@@ -191,6 +203,122 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) th return builder.build(); } + @Override + public Reference createReference(Reference reference, Path path) throws IOException { + FSDataOutputStream out = ctx.getRegionFileSystem().getFileSystem().create(path, false); + try { + out.write(reference.toByteArray()); + } finally { + out.close(); + } + return reference; + } + + /** + * Returns true if the specified family has reference files + * @param familyName Column Family Name + * @return true if family contains reference files + */ + public boolean hasReferences() throws IOException { + Path storeDir = ctx.getRegionFileSystem().getStoreDir(ctx.getFamily().getNameAsString()); + FileStatus[] files = + CommonFSUtils.listStatus(ctx.getRegionFileSystem().getFileSystem(), storeDir); + if (files != null) { + for (FileStatus stat : files) { + if (stat.isDirectory()) { + continue; + } + if (StoreFileInfo.isReference(stat.getPath())) { + LOG.trace("Reference {}", stat.getPath()); + return true; + } + } + } + return false; + } + + @Override + public Reference readReference(final Path p) throws IOException { + InputStream in = ctx.getRegionFileSystem().getFileSystem().open(p); + try { + // I need to be able to move back in the stream if this is not a pb serialization so I can + // do the Writable decoding instead. + in = in.markSupported() ? in : new BufferedInputStream(in); + int pblen = ProtobufUtil.lengthOfPBMagic(); + in.mark(pblen); + byte[] pbuf = new byte[pblen]; + IOUtils.readFully(in, pbuf, 0, pblen); + // WATCHOUT! Return in middle of function!!! + if (ProtobufUtil.isPBMagicPrefix(pbuf)) { + return Reference.convert( + org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference.parseFrom(in)); + } + // Else presume Writables. Need to reset the stream since it didn't start w/ pb. + // We won't bother rewriting thie Reference as a pb since Reference is transitory. + in.reset(); + Reference r = new Reference(); + DataInputStream dis = new DataInputStream(in); + // Set in = dis so it gets the close below in the finally on our way out. + in = dis; + r.readFields(dis); + return r; + } finally { + in.close(); + } + } + + @Override + public StoreFileInfo getStoreFileInfo(Path initialPath, boolean primaryReplica) + throws IOException { + return getStoreFileInfo(null, initialPath, primaryReplica); + } + + @Override + public StoreFileInfo getStoreFileInfo(FileStatus fileStatus, Path initialPath, + boolean primaryReplica) throws IOException { + FileSystem fs = this.ctx.getRegionFileSystem().getFileSystem(); + assert fs != null; + assert initialPath != null; + assert conf != null; + Reference reference = null; + HFileLink link = null; + long createdTimestamp = 0; + long size = 0; + Path p = initialPath; + if (HFileLink.isHFileLink(p)) { + // HFileLink + reference = null; + link = HFileLink.buildFromHFileLinkPattern(conf, p); + LOG.trace("{} is a link", p); + } else if (StoreFileInfo.isReference(p)) { + reference = readReference(p); + Path referencePath = StoreFileInfo.getReferredToFile(p); + if (HFileLink.isHFileLink(referencePath)) { + // HFileLink Reference + link = HFileLink.buildFromHFileLinkPattern(conf, referencePath); + } else { + // Reference + link = null; + } + LOG.trace("{} is a {} reference to {}", p, reference.getFileRegion(), referencePath); + } else + if (StoreFileInfo.isHFile(p) || StoreFileInfo.isMobFile(p) || StoreFileInfo.isMobRefFile(p)) { + // HFile + if (fileStatus != null) { + createdTimestamp = fileStatus.getModificationTime(); + size = fileStatus.getLen(); + } else { + FileStatus fStatus = fs.getFileStatus(initialPath); + createdTimestamp = fStatus.getModificationTime(); + size = fStatus.getLen(); + } + } else { + throw new IOException("path=" + p + " doesn't look like a valid StoreFile"); + } + return new StoreFileInfo(conf, fs, createdTimestamp, initialPath, size, reference, link, + isPrimaryReplica); + } + /** * For primary replica, we will call load once when opening a region, and the implementation could * choose to do some cleanup work. So here we use {@code readOnly} to indicate that whether you diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 0f487afd1cba..828f1974fca7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -129,10 +129,16 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli */ public static StoreFileTracker create(Configuration conf, TableDescriptor td, ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) { + return create(conf, td, cfd, regionFs, true); + } + + public static StoreFileTracker create(Configuration conf, TableDescriptor td, + ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs, boolean isPrimaryReplica) { StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs) .withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build(); - return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), true, ctx); + return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), isPrimaryReplica, + ctx); } private static Configuration mergeConfigurations(Configuration global, TableDescriptor table, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index dab581041ee5..958e2d6faabb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; import org.apache.hadoop.hbase.security.access.TablePermission; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.RestoreMetaChanges; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -500,6 +501,9 @@ private void restoreRegion(final RegionInfo regionInfo, for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { byte[] family = Bytes.toBytes(familyDir.getName()); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withColumnFamilyDescriptor(tableDesc.getColumnFamily(family)) + .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); Set familyFiles = getTableRegionFamilyFiles(familyDir); List snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); @@ -512,7 +516,7 @@ private void restoreRegion(final RegionInfo regionInfo, familyFiles.remove(storeFile.getName()); // no need to restore already present files, but we need to add those to tracker filesToTrack - .add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true)); + .add(tracker.getStoreFileInfo(new Path(familyDir, storeFile.getName()), true)); } else { // HFile missing hfilesToAdd.add(storeFile); @@ -533,9 +537,10 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) { LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); - String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + String fileName = + restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); // mark the reference file to be added to tracker - filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); + filesToTrack.add(tracker.getStoreFileInfo(new Path(familyDir, fileName), true)); } } else { // Family doesn't exists in the snapshot @@ -545,10 +550,6 @@ private void restoreRegion(final RegionInfo regionInfo, fs.delete(familyDir, true); } - StoreFileTracker tracker = - StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder() - .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); - // simply reset list of tracked files with the matching files // and the extra one present in the snapshot tracker.set(filesToTrack); @@ -569,14 +570,14 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) { LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName); - String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); - files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); + String fileName = + restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); + files.add(tracker.getStoreFileInfo(new Path(familyDir, fileName), true)); } tracker.set(files); } } - /** Returns The set of files in the specified family directory. */ private Set getTableRegionFamilyFiles(final Path familyDir) throws IOException { FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir); if (hfiles == null) { @@ -659,6 +660,16 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); List clonedFiles = new ArrayList<>(); + Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); + HRegionFileSystem regionFS = (fs.exists(regionPath)) + ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) + : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); + + Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, + tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName); @@ -669,24 +680,15 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, if (fs.exists(mobPath)) { fs.delete(mobPath, true); } - restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); } else { - String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); - clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); + String file = + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + clonedFiles.add(tracker.getStoreFileInfo(new Path(familyDir, file), true)); } } // we don't need to track files under mobdir if (!MobUtils.isMobRegionInfo(newRegionInfo)) { - Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); - HRegionFileSystem regionFS = (fs.exists(regionPath)) - ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) - : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); - - Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, - tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); - StoreFileTracker tracker = - StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder() - .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); tracker.set(clonedFiles); } } @@ -720,13 +722,13 @@ private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionIn * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, - final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) - throws IOException { + final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef, + final StoreFileTracker tracker) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); } else if (StoreFileInfo.isReference(hfileName)) { - return restoreReferenceFile(familyDir, regionInfo, storeFile); + return restoreReferenceFile(familyDir, regionInfo, storeFile, tracker); } else { return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); } @@ -756,7 +758,8 @@ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInf * @param storeFile reference file name */ private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + final SnapshotRegionManifest.StoreFile storeFile, final StoreFileTracker tracker) + throws IOException { String hfileName = storeFile.getName(); // Extract the referred information (hfile name and parent region) @@ -790,7 +793,7 @@ private String restoreReferenceFile(final Path familyDir, final RegionInfo regio // Create the new reference if (storeFile.hasReference()) { Reference reference = Reference.convert(storeFile.getReference()); - reference.write(fs, outPath); + tracker.createReference(reference, outPath); } else { InputStream in; if (linkPath != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 7dbf5b0655e0..a53d15d2f8e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -207,7 +208,7 @@ protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws monitor.rethrowException(); Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString()); - List storeFiles = getStoreFiles(storePath); + List storeFiles = getStoreFiles(storePath, htd, hcd, regionInfo); if (storeFiles == null) { if (LOG.isDebugEnabled()) { LOG.debug("No mob files under family: " + hcd.getNameAsString()); @@ -341,13 +342,18 @@ protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor vis } } - private List getStoreFiles(Path storeDir) throws IOException { - FileStatus[] stats = CommonFSUtils.listStatus(rootFs, storeDir); + private List getStoreFiles(Path storePath, TableDescriptor htd, + ColumnFamilyDescriptor hcd, RegionInfo regionInfo) throws IOException { + FileStatus[] stats = CommonFSUtils.listStatus(rootFs, storePath); if (stats == null) return null; + HRegionFileSystem regionFS = HRegionFileSystem.create(conf, rootFs, + MobUtils.getMobTableDir(new Path(conf.get(HConstants.HBASE_DIR)), htd.getTableName()), + regionInfo); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false); ArrayList storeFiles = new ArrayList<>(stats.length); for (int i = 0; i < stats.length; ++i) { - storeFiles.add(new StoreFileInfo(conf, rootFs, stats[i])); + storeFiles.add(sft.getStoreFileInfo(stats[i], stats[i].getPath(), false)); } return storeFiles; } @@ -385,7 +391,7 @@ private void load() throws IOException { ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader"); try { this.regionManifests = - SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, htd); } finally { tpool.shutdown(); } @@ -403,7 +409,7 @@ private void load() throws IOException { ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader"); try { v1Regions = - SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, htd); v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, manifestSizeLimit); } catch (InvalidProtocolBufferException e) { @@ -502,7 +508,7 @@ private void convertToV2SingleManifest() throws IOException { setStatusMsg("Loading Region manifests for " + this.desc.getName()); try { v1Regions = - SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, htd); v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, manifestSizeLimit); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java index 61c366de971a..26aeadc41928 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java @@ -30,9 +30,13 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -119,7 +123,7 @@ public void storeFile(final HRegionFileSystem region, final Path familyDir, static List loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, - final SnapshotDescription desc) throws IOException { + final SnapshotDescription desc, final TableDescriptor htd) throws IOException { FileStatus[] regions = CommonFSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs)); if (regions == null) { @@ -134,7 +138,7 @@ static List loadRegionManifests(final Configuration conf @Override public SnapshotRegionManifest call() throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath()); - return buildManifestFromDisk(conf, fs, snapshotDir, hri); + return buildManifestFromDisk(conf, fs, snapshotDir, hri, htd); } }); } @@ -159,7 +163,8 @@ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir, } static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, - final Path tableDir, final RegionInfo regionInfo) throws IOException { + final Path tableDir, final RegionInfo regionInfo, final TableDescriptor htd) + throws IOException { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true); SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder(); @@ -179,7 +184,9 @@ static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, fi Collection familyNames = regionFs.getFamilies(); if (familyNames != null) { for (String familyName : familyNames) { - Collection storeFiles = regionFs.getStoreFiles(familyName, false); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, + htd.getColumnFamily(familyName.getBytes()), regionFs, false); + List storeFiles = getStoreFiles(sft, regionFs, familyName, false); if (storeFiles == null) { LOG.debug("No files under family: " + familyName); continue; @@ -210,4 +217,32 @@ static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, fi } return manifest.build(); } + + public static List getStoreFiles(StoreFileTracker sft, HRegionFileSystem regionFS, + String familyName, boolean validate) throws IOException { + Path familyDir = new Path(regionFS.getRegionDir(), familyName); + FileStatus[] files = CommonFSUtils.listStatus(regionFS.getFileSystem(), familyDir); + if (files == null) { + if (LOG.isTraceEnabled()) { + LOG.trace("No StoreFiles for: " + familyDir); + } + return null; + } + + ArrayList storeFiles = new ArrayList<>(files.length); + for (FileStatus status : files) { + if (validate && !StoreFileInfo.isValid(status)) { + // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to + // true, refer HBASE-23740 + if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) { + LOG.warn("Invalid StoreFile: {}", status.getPath()); + } + continue; + } + StoreFileInfo info = sft.getStoreFileInfo(status.getPath(), false); + storeFiles.add(info); + + } + return storeFiles; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java index 1e9d30a27883..28b99903aac4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint; @@ -123,12 +124,12 @@ public static boolean shouldReplayRecoveredEdits(HRegion region) { * archive after compaction */ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, - RegionInfo regionInfo, RegionInfo regionInfoForFs, String familyName, Path path) - throws IOException { + RegionInfo regionInfo, RegionInfo regionInfoForFs, String familyName, Path path, + StoreFileTracker tracker) throws IOException { // if this is a primary region, just return the StoreFileInfo constructed from path if (RegionInfo.COMPARATOR.compare(regionInfo, regionInfoForFs) == 0) { - return new StoreFileInfo(conf, fs, path, true); + return tracker.getStoreFileInfo(path, true); } // else create a store file link. The link file does not exists on filesystem though. @@ -137,7 +138,7 @@ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, regionInfoForFs.getEncodedName(), familyName, path.getName()); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } else if (StoreFileInfo.isReference(path)) { - Reference reference = Reference.read(fs, path); + Reference reference = tracker.readReference(path); Path referencePath = StoreFileInfo.getReferredToFile(path); if (HFileLink.isHFileLink(referencePath)) { // HFileLink Reference diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java index 31aded84109c..38d45f0f4653 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java @@ -28,8 +28,11 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -100,14 +103,15 @@ Set getStoresRequiringCompaction(Set requestedStores, long times boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { // do we have any store files? - Collection storeFiles = fileSystem.getStoreFiles(family); + StoreFileTracker sft = getStoreFileTracker(family, fileSystem); + List storeFiles = sft.load(); if (storeFiles == null) { LOG.info("Excluding store: " + family + " for compaction for region: " + fileSystem.getRegionInfo().getEncodedName(), " has no store files"); return false; } // check for reference files - if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) { + if (sft.hasReferences() && familyHasReferenceFile(fileSystem, family, ts)) { LOG.info("Including store: " + family + " with: " + storeFiles.size() + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName()); return true; @@ -121,6 +125,13 @@ boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts return includeStore; } + public StoreFileTracker getStoreFileTracker(String family, HRegionFileSystem fileSystem) + throws IOException { + TableDescriptor htd = connection.getTable(getRegion().getTable()).getDescriptor(); + return StoreFileTrackerFactory.create(connection.getConfiguration(), htd, + htd.getColumnFamily(family.getBytes()), fileSystem, false); + } + protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family, Collection storeFiles, long ts) throws IOException { diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp index b538cb7b6b4f..290dcd0342b4 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp @@ -54,7 +54,7 @@ printer.setConf(conf); String[] options = {"-s"}; printer.parseOptions(options); - StoreFileInfo sfi = new StoreFileInfo(conf, fs, new Path(storeFile), true); + StoreFileInfo sfi = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, new Path(storeFile), true); printer.processFile(sfi.getFileStatus().getPath(), true); String text = byteStream.toString();%> <%= diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java index 06373cda4023..8cc568c130b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java @@ -44,6 +44,9 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; @@ -509,7 +512,20 @@ public void testMergeRegion() throws Exception { Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName); HRegionFileSystem regionFs = HRegionFileSystem .openRegionFromFileSystem(UTIL.getConfiguration(), fs, tableDir, mergedRegion, true); - return !regionFs.hasReferences(admin.getDescriptor(tableName)); + boolean references = false; + Path regionDir = new Path(tableDir, mergedRegion.getEncodedName()); + for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { + StoreContext storeContext = StoreContext.getBuilder() + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(familyDir.getName())) + .withRegionFileSystem(regionFs).withFamilyStoreDirectoryPath(familyDir).build(); + StoreFileTracker sft = + StoreFileTrackerFactory.create(UTIL.getConfiguration(), false, storeContext); + references = references || sft.hasReferences(); + if (references) { + break; + } + } + return !references; } catch (IOException e) { LOG.warn("Failed check merged region has no reference", e); return false; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java index 0ac03b8d4136..a999a4ac879c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -35,6 +36,10 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -42,8 +47,12 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -83,14 +92,18 @@ public static void tearDownAfterClass() throws Exception { * top of the file while we are at it. */ @Test - public void testHalfScanAndReseek() throws IOException { + public void testHalfScanAndReseek() throws IOException, InterruptedException { Configuration conf = TEST_UTIL.getConfiguration(); FileSystem fs = FileSystem.get(conf); String root_dir = TEST_UTIL.getDataTestDir().toString(); Path parentPath = new Path(new Path(root_dir, "parent"), "CF"); fs.mkdirs(parentPath); - Path splitAPath = new Path(new Path(root_dir, "splita"), "CF"); - Path splitBPath = new Path(new Path(root_dir, "splitb"), "CF"); + String tableName = Paths.get(root_dir).getFileName().toString(); + RegionInfo splitAHri = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); + Thread.currentThread().sleep(1000); + RegionInfo splitBHri = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); + Path splitAPath = new Path(new Path(root_dir, splitAHri.getRegionNameAsString()), "CF"); + Path splitBPath = new Path(new Path(root_dir, splitBHri.getRegionNameAsString()), "CF"); Path filePath = StoreFileWriter.getUniqueFile(fs, parentPath); CacheConfig cacheConf = new CacheConfig(conf); @@ -112,12 +125,24 @@ public void testHalfScanAndReseek() throws IOException { Path splitFileA = new Path(splitAPath, filePath.getName() + ".parent"); Path splitFileB = new Path(splitBPath, filePath.getName() + ".parent"); + HRegionFileSystem splitAregionFS = + HRegionFileSystem.create(conf, fs, new Path(root_dir), splitAHri); + StoreContext splitAStoreContext = + StoreContext.getBuilder().withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("CF")) + .withFamilyStoreDirectoryPath(splitAPath).withRegionFileSystem(splitAregionFS).build(); + StoreFileTracker splitAsft = StoreFileTrackerFactory.create(conf, false, splitAStoreContext); Reference bottom = new Reference(midkey, Reference.Range.bottom); - bottom.write(fs, splitFileA); + splitAsft.createReference(bottom, splitFileA); doTestOfScanAndReseek(splitFileA, fs, bottom, cacheConf); + HRegionFileSystem splitBregionFS = + HRegionFileSystem.create(conf, fs, new Path(root_dir), splitBHri); + StoreContext splitBStoreContext = + StoreContext.getBuilder().withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("CF")) + .withFamilyStoreDirectoryPath(splitBPath).withRegionFileSystem(splitBregionFS).build(); + StoreFileTracker splitBsft = StoreFileTrackerFactory.create(conf, false, splitBStoreContext); Reference top = new Reference(midkey, Reference.Range.top); - top.write(fs, splitFileB); + splitBsft.createReference(top, splitFileB); doTestOfScanAndReseek(splitFileB, fs, top, cacheConf); r.close(); @@ -133,7 +158,8 @@ private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, Cach new ReaderContextBuilder().withInputStreamWrapper(in).withFileSize(length) .withReaderType(ReaderContext.ReaderType.PREAD).withFileSystem(fs).withFilePath(p); ReaderContext context = contextBuilder.build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, p, true); + StoreFileInfo storeFileInfo = + new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, fs.getFileStatus(p), bottom); storeFileInfo.initHFileInfo(context); final HalfStoreFileReader halfreader = (HalfStoreFileReader) storeFileInfo.createReader(context, cacheConf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java index 7e7b4cb5c37c..d90a48a4be98 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBytesReadFromFs.java @@ -331,7 +331,7 @@ private void readBloomFilters(Path path, BloomType bt, byte[] key, KeyValue keyV readLoadOnOpenDataSection(path, true); CacheConfig cacheConf = new CacheConfig(conf); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, path, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, path, true); HStoreFile sf = new HStoreFile(storeFileInfo, bt, cacheConf); // Read HFile trailer and load-on-open data section diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index c9966745d717..8bbb14fd966b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -69,9 +69,12 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.PrefetchExecutorNotifier; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.TestHStoreFile; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.trace.TraceUtil; @@ -383,11 +386,14 @@ private void testPrefetchWhenRefs(boolean compactionEnabled, Consumer Path storeFile = fileWithSplitPoint.getFirst(); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, region); - HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(new Path(regionDir, "cf")) + .withRegionFileSystem(regionFS).build()); + HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true, sft); Path ref = regionFS.splitStoreFile(region, "cf", file, fileWithSplitPoint.getSecond(), false, - new ConstantSizeRegionSplitPolicy()); + new ConstantSizeRegionSplitPolicy(), sft); conf.setBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, compactionEnabled); - HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true); + HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true, sft); refHsf.initReader(); HFile.Reader reader = refHsf.getReader().getHFileReader(); while (!reader.prefetchComplete()) { @@ -424,13 +430,21 @@ private void testPrefetchWhenHFileLink(Consumer test) throws Exceptio Bytes.toBytes("testPrefetchWhenHFileLink")); Path storeFilePath = regionFs.commitStoreFile("cf", writer.getPath()); - Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", "cf")); + final RegionInfo dstHri = + RegionInfoBuilder.newBuilder(TableName.valueOf("testPrefetchWhenHFileLink")).build(); + HRegionFileSystem dstRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, dstHri.getTable()), dstHri); + Path dstPath = new Path(regionFs.getTableDir(), new Path(dstHri.getRegionNameAsString(), "cf")); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName())); + StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(dstRegionFs.getRegionDir(), "cf")) + .withRegionFileSystem(dstRegionFs).build()); // Try to open store file from link - StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true); + StoreFileInfo storeFileInfo = sft.getStoreFileInfo(linkFilePath, true); HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); assertTrue(storeFileInfo.isLink()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index 1e572e8c55e0..15fc42656ad4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -54,7 +54,10 @@ import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -163,7 +166,9 @@ public void testPrefetchRefsAfterSplit() throws Exception { HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, region); Path storeFile = writeStoreFile(100, cfDir); - + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withRegionFileSystem(regionFS).withFamilyStoreDirectoryPath(cfDir) + .withCacheConfig(cacheConf).build()); // Prefetches the file blocks LOG.debug("First read should prefetch the blocks."); readStoreFile(storeFile); @@ -174,10 +179,10 @@ public void testPrefetchRefsAfterSplit() throws Exception { // split the file and return references to the original file Random rand = ThreadLocalRandom.current(); byte[] splitPoint = RandomKeyValueUtil.randomOrderedKey(rand, 50); - HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true); + HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true, sft); Path ref = regionFS.splitStoreFile(region, "cf", file, splitPoint, false, - new ConstantSizeRegionSplitPolicy()); - HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true); + new ConstantSizeRegionSplitPolicy(), sft); + HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true, sft); // starts reader for the ref. The ref should resolve to the original file blocks // and not duplicate blocks in the cache. refHsf.initReader(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java index c2e24d1f569d..d713727460c8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.MetaMockingUtil; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -60,6 +61,9 @@ import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.MemStoreLAB; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -134,8 +138,10 @@ public void testCleanMerge() throws IOException { Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName()); - Path parentaRef = createMergeReferenceFile(storedir, merged, parenta); - Path parentbRef = createMergeReferenceFile(storedir, merged, parentb); + Path parentaRef = + createMergeReferenceFile(storedir, tabledir, td.getColumnFamilies()[0], merged, parenta); + Path parentbRef = + createMergeReferenceFile(storedir, tabledir, td.getColumnFamilies()[0], merged, parentb); // references exist, should not clean assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents)); @@ -170,7 +176,7 @@ public void testDontCleanMergeIfFileSystemException() throws IOException { Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName()); Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName()); - createMergeReferenceFile(storedir, merged, parenta); + createMergeReferenceFile(storedir, tabledir, td.getColumnFamilies()[0], merged, parenta); MasterServices mockedMasterServices = spy(masterServices); MasterFileSystem mockedMasterFileSystem = spy(masterServices.getMasterFileSystem()); @@ -198,14 +204,22 @@ public void testDontCleanMergeIfFileSystemException() throws IOException { assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents)); } - private Path createMergeReferenceFile(Path storeDir, HRegionInfo mergedRegion, - HRegionInfo parentRegion) throws IOException { + private Path createMergeReferenceFile(Path storeDir, Path tableDir, + ColumnFamilyDescriptor columnFamilyDescriptor, RegionInfo mergedRegion, RegionInfo parentRegion) + throws IOException { Reference ref = Reference.createTopReference(mergedRegion.getStartKey()); long now = EnvironmentEdgeManager.currentTime(); // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storeDir, Long.toString(now) + "." + parentRegion.getEncodedName()); FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem(); - return ref.write(fs, p); + HRegionFileSystem mergedRegionFS = + HRegionFileSystem.create(fs.getConf(), fs, tableDir, mergedRegion); + StoreContext storeContext = + StoreContext.getBuilder().withColumnFamilyDescriptor(columnFamilyDescriptor) + .withFamilyStoreDirectoryPath(storeDir).withRegionFileSystem(mergedRegionFS).build(); + StoreFileTracker sft = StoreFileTrackerFactory.create(fs.getConf(), false, storeContext); + sft.createReference(ref, p); + return p; } /** @@ -234,9 +248,16 @@ public void testCleanParent() throws IOException, InterruptedException { // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem(); - Path path = ref.write(fs, p); - assertTrue(fs.exists(path)); - LOG.info("Created reference " + path); + HRegionFileSystem regionFS = + HRegionFileSystem.create(this.masterServices.getConfiguration(), fs, tabledir, splita); + StoreContext storeContext = + StoreContext.getBuilder().withColumnFamilyDescriptor(td.getColumnFamilies()[0]) + .withFamilyStoreDirectoryPath(storedir).withRegionFileSystem(regionFS).build(); + StoreFileTracker sft = + StoreFileTrackerFactory.create(this.masterServices.getConfiguration(), false, storeContext); + sft.createReference(ref, p); + assertTrue(fs.exists(p)); + LOG.info("Created reference " + p); // Add a parentdir for kicks so can check it gets removed by the catalogjanitor. fs.mkdirs(parentdir); assertFalse(CatalogJanitor.cleanParent(masterServices, parent, r)); @@ -704,7 +725,14 @@ private Path createReferences(final MasterServices services, final TableDescript // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); FileSystem fs = services.getMasterFileSystem().getFileSystem(); - ref.write(fs, p); + HRegionFileSystem regionFS = + HRegionFileSystem.create(services.getConfiguration(), fs, tabledir, daughter); + StoreContext storeContext = + StoreContext.getBuilder().withColumnFamilyDescriptor(td.getColumnFamilies()[0]) + .withFamilyStoreDirectoryPath(storedir).withRegionFileSystem(regionFS).build(); + StoreFileTracker sft = + StoreFileTrackerFactory.create(services.getConfiguration(), false, storeContext); + sft.createReference(ref, p); return p; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java index 9c45a62aed34..eee9fd0d5fc9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java @@ -30,6 +30,9 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -69,8 +72,11 @@ public void testOpenClose() throws Exception { HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs).withOutputDir(testDir) .withFileContext(meta).build(); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); MobTestUtil.writeStoreFile(writer, caseName); - CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf); + CachedMobFile cachedMobFile = + new CachedMobFile(new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf)); assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount()); cachedMobFile.open(); assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount()); @@ -93,12 +99,18 @@ public void testCompare() throws Exception { StoreFileWriter writer1 = new StoreFileWriter.Builder(conf, cacheConf, fs) .withOutputDir(outputDir1).withFileContext(meta).build(); MobTestUtil.writeStoreFile(writer1, caseName); - CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf); + StoreFileInfo storeFileInfo1 = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer1.getPath(), true); + CachedMobFile cachedMobFile1 = + new CachedMobFile(new HStoreFile(storeFileInfo1, BloomType.NONE, cacheConf)); Path outputDir2 = new Path(testDir, FAMILY2); StoreFileWriter writer2 = new StoreFileWriter.Builder(conf, cacheConf, fs) .withOutputDir(outputDir2).withFileContext(meta).build(); MobTestUtil.writeStoreFile(writer2, caseName); - CachedMobFile cachedMobFile2 = CachedMobFile.create(fs, writer2.getPath(), conf, cacheConf); + StoreFileInfo storeFileInfo2 = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer2.getPath(), true); + CachedMobFile cachedMobFile2 = + new CachedMobFile(new HStoreFile(storeFileInfo2, BloomType.NONE, cacheConf)); cachedMobFile1.access(1); cachedMobFile2.access(2); assertEquals(1, cachedMobFile1.compareTo(cachedMobFile2)); @@ -115,7 +127,10 @@ public void testReadKeyValue() throws Exception { .withFileContext(meta).build(); String caseName = testName.getMethodName(); MobTestUtil.writeStoreFile(writer, caseName); - CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + CachedMobFile cachedMobFile = + new CachedMobFile(new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf)); byte[] family = Bytes.toBytes(caseName); byte[] qualify = Bytes.toBytes(caseName); // Test the start key diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java index 6aeab33893f2..a59b839a806f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; import org.junit.After; @@ -42,6 +43,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category(MediumTests.class) public class TestExpiredMobFileCleaner { @@ -57,6 +60,7 @@ public class TestExpiredMobFileCleaner { private final static byte[] row2 = Bytes.toBytes("row2"); private final static byte[] row3 = Bytes.toBytes("row3"); private final static byte[] qf = Bytes.toBytes("qf"); + private static final Logger LOG = LoggerFactory.getLogger(TestExpiredMobFileCleaner.class); private static BufferedMutator table; private static Admin admin; @@ -136,6 +140,9 @@ public void testCleaner() throws Exception { byte[] dummyData = makeDummyData(600); long ts = EnvironmentEdgeManager.currentTime() - 3 * secondsOfDay() * 1000; // 3 days before putKVAndFlush(table, row1, dummyData, ts); + LOG.info("test log to be deleted, tablename is " + tableName); + CommonFSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), + TEST_UTIL.getDefaultRootDirPath(), LOG); FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); // the first mob file assertEquals("Before cleanup without delay 1", 1, firstFiles.length); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java index 5e593c8c5e1a..ac7645bf1d9d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -67,8 +68,9 @@ public void testReadKeyValue() throws Exception { String caseName = testName.getMethodName(); MobTestUtil.writeStoreFile(writer, caseName); - MobFile mobFile = - new MobFile(new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true)); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + MobFile mobFile = new MobFile(new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf)); byte[] family = Bytes.toBytes(caseName); byte[] qualify = Bytes.toBytes(caseName); @@ -116,8 +118,9 @@ public void testGetScanner() throws Exception { .withFileContext(meta).build(); MobTestUtil.writeStoreFile(writer, testName.getMethodName()); - MobFile mobFile = - new MobFile(new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true)); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + MobFile mobFile = new MobFile(new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf)); assertNotNull(mobFile.getScanner()); assertTrue(mobFile.getScanner() instanceof StoreFileScanner); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java index 069fc48322bb..773203300a8f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java @@ -26,21 +26,24 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.HMobStore; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -116,37 +119,21 @@ public void tearDown() throws Exception { /** * Create the mob store file. */ - private Path createMobStoreFile(String family) throws IOException { - return createMobStoreFile(HBaseConfiguration.create(), family); - } - - /** - * Create the mob store file - */ - private Path createMobStoreFile(Configuration conf, String family) throws IOException { - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMaxVersions(4); - hcd.setMobEnabled(true); - return createMobStoreFile(hcd); - } - - /** - * Create the mob store file - */ - private Path createMobStoreFile(HColumnDescriptor hcd) throws IOException { + private Pair createAndGetMobStoreFileContextPair(String family) + throws IOException { + ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder + .newBuilder(Bytes.toBytes(family)).setMaxVersions(4).setMobEnabled(true).build(); // Setting up a Store TableName tn = TableName.valueOf(TABLE); - HTableDescriptor htd = new HTableDescriptor(tn); - htd.addFamily(hcd); - HMobStore mobStore = (HMobStore) region.getStore(hcd.getName()); - KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE); - KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE); - KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2); + HMobStore mobStore = (HMobStore) region.getStore(columnFamilyDescriptor.getName()); + KeyValue key1 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF1, 1, VALUE); + KeyValue key2 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF2, 1, VALUE); + KeyValue key3 = new KeyValue(ROW2, columnFamilyDescriptor.getName(), QF3, 1, VALUE2); KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; int maxKeyCount = keys.length; HRegionInfo regionInfo = new HRegionInfo(tn); StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate, maxKeyCount, - hcd.getCompactionCompression(), regionInfo.getStartKey(), false); + columnFamilyDescriptor.getCompactionCompressionType(), regionInfo.getStartKey(), false); Path mobFilePath = mobWriter.getPath(); String fileName = mobFilePath.getName(); mobWriter.append(key1); @@ -156,21 +143,26 @@ private Path createMobStoreFile(HColumnDescriptor hcd) throws IOException { String targetPathName = MobUtils.formatDate(currentDate); Path targetPath = new Path(mobStore.getPath(), targetPathName); mobStore.commitFile(mobFilePath, targetPath); - return new Path(targetPath, fileName); + return new Pair(new Path(targetPath, fileName), mobStore.getStoreContext()); } @Test public void testMobFileCache() throws Exception { FileSystem fs = FileSystem.get(conf); - Path file1Path = createMobStoreFile(FAMILY1); - Path file2Path = createMobStoreFile(FAMILY2); - Path file3Path = createMobStoreFile(FAMILY3); + Pair fileAndContextPair1 = createAndGetMobStoreFileContextPair(FAMILY1); + Pair fileAndContextPair2 = createAndGetMobStoreFileContextPair(FAMILY2); + Pair fileAndContextPair3 = createAndGetMobStoreFileContextPair(FAMILY3); + + Path file1Path = fileAndContextPair1.getFirst(); + Path file2Path = fileAndContextPair2.getFirst(); + Path file3Path = fileAndContextPair3.getFirst(); CacheConfig cacheConf = new CacheConfig(conf); // Before open one file by the MobFileCache assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize()); // Open one file by the MobFileCache - CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf); + CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf, + fileAndContextPair1.getSecond()); assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); assertNotNull(cachedMobFile1); assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount()); @@ -189,11 +181,14 @@ public void testMobFileCache() throws Exception { cachedMobFile1.close(); // Close the cached mob file // Reopen three cached file - cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf); + cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(fs, file1Path, cacheConf, + fileAndContextPair1.getSecond()); assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize()); - CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(fs, file2Path, cacheConf); + CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(fs, file2Path, cacheConf, + fileAndContextPair2.getSecond()); assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize()); - CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(fs, file3Path, cacheConf); + CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(fs, file3Path, cacheConf, + fileAndContextPair3.getSecond()); // Before the evict // Evict the cache, should close the first file 1 assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java index 123965c0eca2..bdfb42a14970 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java @@ -62,12 +62,15 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.RegionAsTable; +import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.security.User; @@ -329,9 +332,17 @@ private long countMobCellsInMetadata() throws IOException { copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); if (fs.exists(mobDirPath)) { + // TODO: use sft.load() api here + HRegionFileSystem regionFs = HRegionFileSystem.create(copyOfConf, fs, + MobUtils.getMobTableDir(copyOfConf, htd.getTableName()), region.getRegionInfo()); + StoreFileTracker sft = StoreFileTrackerFactory.create(copyOfConf, false, + StoreContext.getBuilder().withColumnFamilyDescriptor(hcd) + .withFamilyStoreDirectoryPath(mobDirPath).withCacheConfig(cacheConfig) + .withRegionFileSystem(regionFs).build()); FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); for (FileStatus file : files) { - HStoreFile sf = new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true); + HStoreFile sf = + new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true, sft); sf.initReader(); Map fileInfo = sf.getReader().loadFileInfo(); byte[] count = fileInfo.get(MOB_CELLS_COUNT); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/AbstractTestDateTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/AbstractTestDateTieredCompactionPolicy.java index 53a9de7f71b5..48a4e7b2984c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/AbstractTestDateTieredCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/AbstractTestDateTieredCompactionPolicy.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; @@ -46,9 +47,11 @@ protected ArrayList sfCreate(long[] minTimestamps, long[] maxTimesta } ArrayList ret = Lists.newArrayList(); + StoreFileTrackerForTest storeFileTrackerForTest = + new StoreFileTrackerForTest(conf, true, store.getStoreContext()); for (int i = 0; i < sizes.length; i++) { - MockHStoreFile msf = - new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i); + MockHStoreFile msf = new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), + false, i, storeFileTrackerForTest); msf.setTimeRangeTracker( TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, minTimestamps[i], maxTimestamps[i])); ret.add(msf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java index ec9de92e9f25..d7a467365477 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java @@ -582,7 +582,8 @@ public static void testCodecs(Configuration conf, int kvLimit, String hfilePath, Path path = new Path(hfilePath); CacheConfig cacheConf = new CacheConfig(conf); FileSystem fs = FileSystem.get(conf); - HStoreFile hsf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, path, true); + HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); hsf.initReader(); StoreFileReader reader = hsf.getReader(); reader.loadFileInfo(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java index fbd6286c5a94..de9494c580a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java @@ -58,8 +58,9 @@ private List prepareListOfTestSeeks(Path path) throws IOException { List allKeyValues = new ArrayList<>(); // read all of the key values - HStoreFile storeFile = new HStoreFile(testingUtility.getTestFileSystem(), path, configuration, - cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(configuration, + testingUtility.getTestFileSystem(), path, true); + HStoreFile storeFile = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false, 0, 0, false); @@ -87,8 +88,9 @@ private List prepareListOfTestSeeks(Path path) throws IOException { private void runTest(Path path, DataBlockEncoding blockEncoding, List seeks) throws IOException { // read all of the key values - HStoreFile storeFile = new HStoreFile(testingUtility.getTestFileSystem(), path, configuration, - cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(configuration, + testingUtility.getTestFileSystem(), path, true); + HStoreFile storeFile = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); storeFile.initReader(); long totalSize = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java index b31c738149f9..4f4717903087 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.Map; import java.util.Optional; @@ -30,6 +31,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.DNS; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -52,9 +54,13 @@ public class MockHStoreFile extends HStoreFile { boolean compactedAway; MockHStoreFile(HBaseTestingUtility testUtil, Path testPath, long length, long ageInDisk, - boolean isRef, long sequenceid) throws IOException { - super(testUtil.getTestFileSystem(), testPath, testUtil.getConfiguration(), - new CacheConfig(testUtil.getConfiguration()), BloomType.NONE, true); + boolean isRef, long sequenceid, StoreFileInfo storeFileInfo) throws IOException { + super(storeFileInfo, BloomType.NONE, new CacheConfig(testUtil.getConfiguration())); + setMockHStoreFileVals(length, isRef, ageInDisk, sequenceid, isMajor, testUtil); + } + + private void setMockHStoreFileVals(long length, boolean isRef, long ageInDisk, long sequenceid, + boolean isMajor, HBaseTestingUtility testUtil) throws UnknownHostException { this.length = length; this.isRef = isRef; this.ageInDisk = ageInDisk; @@ -67,6 +73,13 @@ public class MockHStoreFile extends HStoreFile { modificationTime = EnvironmentEdgeManager.currentTime(); } + MockHStoreFile(HBaseTestingUtility testUtil, Path testPath, long length, long ageInDisk, + boolean isRef, long sequenceid, StoreFileTracker tracker) throws IOException { + super(testUtil.getTestFileSystem(), testPath, testUtil.getConfiguration(), + new CacheConfig(testUtil.getConfiguration()), BloomType.NONE, true, tracker); + setMockHStoreFileVals(length, isRef, ageInDisk, sequenceid, isMajor, testUtil); + } + void setLength(long newLen) { this.length = newLen; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index 22b02b7cf825..6ac3c4e07d23 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -219,7 +219,8 @@ public void testCacheOnWriteInSchema() throws IOException { private void readStoreFile(Path path) throws IOException { CacheConfig cacheConf = store.getCacheConfig(); BlockCache cache = cacheConf.getBlockCache().get(); - HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.ROWCOL, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, path, true); + HStoreFile sf = new HStoreFile(storeFileInfo, BloomType.ROWCOL, cacheConf); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java index 97b62e9d987c..706b98aeef7c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -155,7 +156,10 @@ public void testRemoveCompactedFilesWithException() throws Exception { out.writeInt(1); out.close(); - HStoreFile errStoreFile = new MockHStoreFile(testUtil, errFile, 1, 0, false, 1); + StoreFileTrackerForTest storeFileTrackerForTest = + new StoreFileTrackerForTest(store.getReadOnlyConfiguration(), true, store.getStoreContext()); + HStoreFile errStoreFile = + new MockHStoreFile(testUtil, errFile, 1, 0, false, 1, storeFileTrackerForTest); fileManager.addCompactionResults(ImmutableList.of(errStoreFile), ImmutableList.of()); // cleanup compacted files diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java index dcd900ec33a7..5d764df9eb90 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -162,9 +163,11 @@ List sfCreate(boolean isReference, long... sizes) throws IOException List sfCreate(boolean isReference, ArrayList sizes, ArrayList ageInDisk) throws IOException { List ret = Lists.newArrayList(); + StoreFileTrackerForTest storeFileTrackerForTest = + new StoreFileTrackerForTest(conf, true, store.getStoreContext()); for (int i = 0; i < sizes.size(); i++) { - ret.add( - new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes.get(i), ageInDisk.get(i), isReference, i)); + ret.add(new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes.get(i), ageInDisk.get(i), isReference, + i, storeFileTrackerForTest)); } return ret; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java index 87f9af25e2c7..6f708d26f3a9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java @@ -197,7 +197,8 @@ private void validateFalsePosRate(double falsePosRate, int nTrials, double zValu private void readStoreFile(int t, BloomType bt, List kvs, Path sfPath) throws IOException { - HStoreFile sf = new HStoreFile(fs, sfPath, conf, cacheConf, bt, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, sfPath, true); + HStoreFile sf = new HStoreFile(storeFileInfo, bt, cacheConf); sf.initReader(); StoreFileReader r = sf.getReader(); final boolean pread = true; // does not really matter diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java index 0771d41bb433..a6caff227598 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellDataTieringManager.java @@ -62,6 +62,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -800,7 +802,10 @@ private static HStoreFile createHStoreFile(Path storeDir, Configuration conf, lo writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); - return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreContext storeContext = StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, + sft); } private static Configuration getConfWithCustomCellDataTieringEnabled(long hotDataAge) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java index b2c363bf53ac..3796e3e5b8db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCustomCellTieredCompactionPolicy.java @@ -32,9 +32,11 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.compactions.CustomDateTieredCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -65,7 +67,13 @@ private HStoreFile createFile(RegionInfo regionInfo, Path file, long minValue, l FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); HRegionFileSystem regionFileSystem = new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, file, regionInfo); - MockHStoreFile msf = new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId); + StoreContext ctx = new StoreContext.Builder() + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()) + .withRegionFileSystem(regionFileSystem).build(); + StoreFileTrackerForTest sftForTest = + new StoreFileTrackerForTest(TEST_UTIL.getConfiguration(), true, ctx); + MockHStoreFile msf = + new MockHStoreFile(TEST_UTIL, file, size, ageInDisk, false, (long) seqId, sftForTest); TimeRangeTracker timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); timeRangeTracker.setMin(minValue); timeRangeTracker.setMax(maxValue); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 585482c94093..507f14a86946 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -62,6 +62,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -816,7 +818,11 @@ static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long times writeStoreFileRandomData(storeFileWriter, Bytes.toBytes(columnFamily), timestamp); - return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreContext storeContext = StoreContext.getBuilder().withRegionFileSystem(regionFs).build(); + + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); + return new HStoreFile(fs, storeFileWriter.getPath(), conf, cacheConf, BloomType.NONE, true, + sft); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java index 309b18288669..795262a8486d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDirectStoreSplitsMerges.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -34,6 +35,8 @@ import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -84,8 +87,11 @@ public void testSplitStoreDir() throws Exception { .setRegionId(region.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()) .build(); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; + StoreFileTracker sft = + StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), + true, region.getStores().get(0).getStoreContext()); Path result = regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy()); + Bytes.toBytes("002"), false, region.getSplitPolicy(), sft); // asserts the reference file naming is correct validateResultingFile(region.getRegionInfo().getEncodedName(), result); // Additionally check if split region dir was created directly under table dir, not on .tmp @@ -113,16 +119,18 @@ public void testMergeStoreFile() throws Exception { .setRegionId(first.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()) .build(); - HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem( - TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), regionFS.getFileSystem(), - regionFS.getTableDir(), mergeResult); + Configuration configuration = TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(); + HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem(configuration, + regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); // merge file from first region HStoreFile file = (HStoreFile) first.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, first, file); + mergeFileFromRegion(mergeRegionFs, first, file, StoreFileTrackerFactory.create(configuration, + true, first.getStore(FAMILY_NAME).getStoreContext())); // merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, second, file); + mergeFileFromRegion(mergeRegionFs, second, file, StoreFileTrackerFactory.create(configuration, + true, second.getStore(FAMILY_NAME).getStoreContext())); } @Test @@ -163,11 +171,14 @@ public void testCommitDaughterRegionWithFiles() throws Exception { Path splitDirB = regionFS.getSplitsDir(daughterB); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; List filesA = new ArrayList<>(); + StoreFileTracker sft = + StoreFileTrackerFactory.create(TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), + true, region.getStores().get(0).getStoreContext()); filesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy())); + Bytes.toBytes("002"), false, region.getSplitPolicy(), sft)); List filesB = new ArrayList<>(); filesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), true, region.getSplitPolicy())); + Bytes.toBytes("002"), true, region.getSplitPolicy(), sft)); MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment(); Path resultA = regionFS.commitDaughterRegion(daughterA, filesA, env); @@ -196,17 +207,19 @@ public void testCommitMergedRegion() throws Exception { .setRegionId(first.getRegionInfo().getRegionId() + EnvironmentEdgeManager.currentTime()) .build(); - HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem( - TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(), regionFS.getFileSystem(), - regionFS.getTableDir(), mergeResult); + Configuration configuration = TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(); + HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem(configuration, + regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult); // merge file from first region HStoreFile file = (HStoreFile) first.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - mergeFileFromRegion(mergeRegionFs, first, file); + mergeFileFromRegion(mergeRegionFs, first, file, StoreFileTrackerFactory.create(configuration, + true, first.getStore(FAMILY_NAME).getStoreContext())); // merge file from second region file = (HStoreFile) second.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; List mergedFiles = new ArrayList<>(); - mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file)); + mergedFiles.add(mergeFileFromRegion(mergeRegionFs, second, file, StoreFileTrackerFactory + .create(configuration, true, second.getStore(FAMILY_NAME).getStoreContext()))); MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment(); mergeRegionFs.commitMergedRegion(mergedFiles, env); @@ -229,9 +242,9 @@ private void waitForSplitProcComplete(int attempts, int waitTime) throws Excepti } private Path mergeFileFromRegion(HRegionFileSystem regionFS, HRegion regionToMerge, - HStoreFile file) throws IOException { - Path mergedFile = - regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); + HStoreFile file, StoreFileTracker sft) throws IOException { + Path mergedFile = regionFS.mergeStoreFile(regionToMerge.getRegionInfo(), + Bytes.toString(FAMILY_NAME), file, sft); validateResultingFile(regionToMerge.getRegionInfo().getEncodedName(), mergedFile); return mergedFile; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java index b94025590430..a0886123c16d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java @@ -94,8 +94,9 @@ public void testHFileScannerThrowsErrors() throws IOException { .withOutputDir(hfilePath).withFileContext(meta).build(); TestHStoreFile.writeStoreFile(writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); - HStoreFile sf = new HStoreFile(fs, writer.getPath(), util.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(util.getConfiguration(), + fs, writer.getPath(), true); + HStoreFile sf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); sf.initReader(); StoreFileReader reader = sf.getReader(); HFileScanner scanner = reader.getScanner(false, true); @@ -139,8 +140,9 @@ public void testStoreFileScannerThrowsErrors() throws IOException { .withOutputDir(hfilePath).withFileContext(meta).build(); TestHStoreFile.writeStoreFile(writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); - HStoreFile sf = new HStoreFile(fs, writer.getPath(), util.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(util.getConfiguration(), + fs, writer.getPath(), true); + HStoreFile sf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); List scanners = StoreFileScanner.getScannersForStoreFiles( Collections.singletonList(sf), false, true, false, false, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 059935084066..d9a735144480 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -150,6 +150,8 @@ import org.apache.hadoop.hbase.regionserver.Region.RowLock; import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; @@ -5687,8 +5689,14 @@ public void testCompactionFromPrimary() throws IOException { // move the file of the primary region to the archive, simulating a compaction Collection storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); - Collection storeFileInfos = - primaryRegion.getRegionFileSystem().getStoreFiles(Bytes.toString(families[0])); + HRegionFileSystem regionFs = primaryRegion.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(primaryRegion.getBaseConf(), false, + StoreContext.getBuilder() + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(families[0]).build()) + .withFamilyStoreDirectoryPath( + new Path(regionFs.getRegionDir(), Bytes.toString(families[0]))) + .withRegionFileSystem(regionFs).build()); + Collection storeFileInfos = sft.load(); Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); verifyData(secondaryRegion, 0, 1000, cq, families); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java index 49c2f83e8b3c..7f8e3e71afb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.net.URI; -import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -39,11 +38,14 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -359,20 +361,25 @@ public void testTempAndCommit() throws IOException { ; RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri); - + StoreContext storeContext = StoreContext.getBuilder() + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(familyName)) + .withFamilyStoreDirectoryPath( + new Path(regionFs.getTableDir(), new Path(hri.getRegionNameAsString(), familyName))) + .withRegionFileSystem(regionFs).build(); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext); // New region, no store files - Collection storeFiles = regionFs.getStoreFiles(familyName); + List storeFiles = sft.load(); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); // Create a new file in temp (no files in the family) Path buildPath = regionFs.createTempName(); fs.createNewFile(buildPath); - storeFiles = regionFs.getStoreFiles(familyName); + storeFiles = sft.load(); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); // commit the file Path dstPath = regionFs.commitStoreFile(familyName, buildPath); - storeFiles = regionFs.getStoreFiles(familyName); + storeFiles = sft.load(); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); assertFalse(fs.exists(buildPath)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index eca38d0bc239..5d856edadb01 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -121,6 +121,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor; import org.apache.hadoop.hbase.regionserver.compactions.EverythingPolicy; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; @@ -788,8 +790,8 @@ public Object run() throws Exception { LOG.info("Before flush, we should have no files"); - Collection files = - store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, store.getStoreContext()); + Collection files = sft.load(); assertEquals(0, files != null ? files.size() : 0); // flush @@ -802,7 +804,7 @@ public Object run() throws Exception { } LOG.info("After failed flush, we should still have no files!"); - files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); + files = sft.load(); assertEquals(0, files != null ? files.size() : 0); store.getHRegion().getWAL().close(); return null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java index b93e0472bd71..f21f5ccfcc46 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java @@ -82,6 +82,8 @@ import org.apache.hadoop.hbase.io.hfile.UncompressedBlockSizePredicator; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; @@ -163,8 +165,12 @@ public void testBasicHalfAndHFileLinkMapFile() throws Exception { writeStoreFile(writer); Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); - HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true); - checkHalfHFile(regionFs, sf); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), TEST_FAMILY)) + .withRegionFileSystem(regionFs).build()); + HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true, sft); + checkHalfHFile(regionFs, sf, sft); } private void writeStoreFile(final StoreFileWriter writer) throws IOException { @@ -228,7 +234,11 @@ public void testReference() throws IOException { writeStoreFile(writer); Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); - HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), TEST_FAMILY)) + .withRegionFileSystem(regionFs).build()); + HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true, sft); hsf.initReader(); StoreFileReader reader = hsf.getReader(); // Split on a row, not in middle of row. Midkey returned by reader @@ -239,9 +249,10 @@ public void testReference() throws IOException { hsf.closeStoreFile(true); // Make a reference - HRegionInfo splitHri = new HRegionInfo(hri.getTable(), null, midRow); - Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true); - HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true); + RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build(); + Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true, sft); + HStoreFile refHsf = + new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true, sft); refHsf.initReader(); // Now confirm that I can read from the reference and that it only gets // keys from top half of the file. @@ -274,8 +285,11 @@ public void testStoreFileReference() throws Exception { writeStoreFile(writer); Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); writer.close(); - - HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), TEST_FAMILY)) + .withRegionFileSystem(regionFs).build()); + HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true, sft); file.initReader(); StoreFileReader r = file.getReader(); assertNotNull(r); @@ -312,6 +326,10 @@ public void testHFileLink() throws IOException { CommonFSUtils.setRootDir(testConf, testDir); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); + final RegionInfo dstHri = + RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build(); + HRegionFileSystem dstRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, dstHri.getTable()), dstHri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. @@ -320,13 +338,22 @@ public void testHFileLink() throws IOException { writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); - Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY)); + Path dstPath = + new Path(regionFs.getTableDir(), new Path(dstHri.getRegionNameAsString(), TEST_FAMILY)); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName())); // Try to open store file from link - StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true); + + // this should be the SFT for the destination link file path, though it is not + // being used right now, for the next patch file link creation logic also would + // move to SFT interface. + StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(dstHri.getRegionNameAsString(), TEST_FAMILY)) + .withRegionFileSystem(dstRegionFs).build()); + StoreFileInfo storeFileInfo = sft.getStoreFileInfo(linkFilePath, true); HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); assertTrue(storeFileInfo.isLink()); hsf.initReader(); @@ -341,6 +368,13 @@ public void testHFileLink() throws IOException { assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); } + @Test + public void testsample() { + Path p1 = new Path("/r1/c1"); + Path p2 = new Path("f1"); + System.out.println(new Path(p1, p2).toString()); + } + /** * This test creates an hfile and then the dir structures and files to verify that references to * hfilelinks (created by snapshot clones) can be properly interpreted. @@ -375,12 +409,33 @@ public void testReferenceToHFileLink() throws IOException { // create splits of the link. // /clone/splitA//, // /clone/splitB// - HRegionInfo splitHriA = new HRegionInfo(hri.getTable(), null, SPLITKEY); - HRegionInfo splitHriB = new HRegionInfo(hri.getTable(), SPLITKEY, null); - HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true); + RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build(); + RegionInfo splitHriB = + RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build(); + + StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, true, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(hriClone.getRegionNameAsString(), TEST_FAMILY)) + .withRegionFileSystem(cloneRegionFs).build()); + + HRegionFileSystem splitRegionAFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, splitHriA.getTable()), splitHriA); + StoreFileTracker sftA = StoreFileTrackerFactory.create(testConf, true, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(splitHriA.getRegionNameAsString(), TEST_FAMILY)) + .withRegionFileSystem(splitRegionAFs).build()); + HRegionFileSystem splitRegionBFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, splitHriB.getTable()), splitHriB); + StoreFileTracker sftB = StoreFileTrackerFactory.create(testConf, true, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(splitHriB.getRegionNameAsString(), TEST_FAMILY)) + .withRegionFileSystem(splitRegionBFs).build()); + HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true, sft); f.initReader(); - Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top - Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom + // top + Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true, sft); + // bottom + Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false, sft); f.closeStoreFile(true); // OK test the thing CommonFSUtils.logFileSystemState(fs, testDir, LOG); @@ -389,7 +444,8 @@ public void testReferenceToHFileLink() throws IOException { // reference to a hfile link. This code in StoreFile that handles this case. // Try to open store file from link - HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true); + HStoreFile hsfA = + new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true, sftA); hsfA.initReader(); // Now confirm that I can read from the ref to link @@ -402,7 +458,8 @@ public void testReferenceToHFileLink() throws IOException { assertTrue(count > 0); // read some rows here // Try to open store file from link - HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true); + HStoreFile hsfB = + new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true, sftB); hsfB.initReader(); // Now confirm that I can read from the ref to link @@ -419,8 +476,8 @@ public void testReferenceToHFileLink() throws IOException { assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); } - private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f) - throws IOException { + private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f, + StoreFileTracker sft) throws IOException { f.initReader(); Cell midkey = f.getReader().midKey().get(); KeyValue midKV = (KeyValue) midkey; @@ -428,16 +485,19 @@ private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f // in the children byte[] midRow = CellUtil.cloneRow(midKV); // Create top split. - HRegionInfo topHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), null, midRow); - Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true); + RegionInfo topHri = + RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()).setEndKey(SPLITKEY).build(); + Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true, sft); // Create bottom split. - HRegionInfo bottomHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), midRow, null); - Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false); + RegionInfo bottomHri = RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()) + .setStartKey(SPLITKEY).build(); + Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false, sft); // Make readers on top and bottom. - HStoreFile topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true); + HStoreFile topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true, sft); topF.initReader(); StoreFileReader top = topF.getReader(); - HStoreFile bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true); + HStoreFile bottomF = + new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true, sft); bottomF.initReader(); StoreFileReader bottom = bottomF.getReader(); ByteBuffer previous = null; @@ -493,12 +553,12 @@ private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f // properly. byte[] badmidkey = Bytes.toBytes(" ."); assertTrue(fs.exists(f.getPath())); - topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true); - bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false); + topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true, sft); + bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false, sft); assertNull(bottomPath); - topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true); + topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true, sft); topF.initReader(); top = topF.getReader(); // Now read from the top. @@ -533,11 +593,11 @@ private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f // Test when badkey is > than last key in file ('||' > 'zz'). badmidkey = Bytes.toBytes("|||"); - topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true); - bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false); + topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true, sft); + bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false, sft); assertNull(topPath); - bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true); + bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true, sft); bottomF.initReader(); bottom = bottomF.getReader(); first = true; @@ -592,7 +652,7 @@ private void bloomWriteRead(StoreFileWriter writer, FileSystem fs) throws Except writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -681,7 +741,7 @@ public void testDeleteFamilyBloomFilter() throws Exception { writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -734,7 +794,7 @@ public void testReseek() throws Exception { writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -798,7 +858,7 @@ public void testBloomTypes() throws Exception { ReaderContext context = new ReaderContextBuilder().withFilePath(f).withFileSize(fs.getFileStatus(f).getLen()) .withFileSystem(fs).withInputStreamWrapper(new FSDataInputStreamWrapper(fs, f)).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -936,8 +996,9 @@ public void testMultipleTimestamps() throws IOException { writer.appendMetadata(0, false); writer.close(); - HStoreFile hsf = - new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); HStore store = mock(HStore.class); when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of(family)); hsf.initReader(); @@ -991,13 +1052,14 @@ public void testCacheOnWriteEvictOnClose() throws Exception { CacheConfig cacheConf = new CacheConfig(conf, bc); Path pathCowOff = new Path(baseDir, "123456789"); StoreFileWriter writer = writeStoreFile(conf, cacheConf, pathCowOff, 3); - HStoreFile hsf = - new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); - LOG.debug(hsf.getPath().toString()); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile hsfCowOff = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); + LOG.debug(hsfCowOff.getPath().toString()); // Read this file, we should see 3 misses - hsf.initReader(); - StoreFileReader reader = hsf.getReader(); + hsfCowOff.initReader(); + StoreFileReader reader = hsfCowOff.getReader(); reader.loadFileInfo(); StoreFileScanner scanner = getStoreFileScanner(reader, true, true); scanner.seek(KeyValue.LOWESTKEY); @@ -1016,11 +1078,12 @@ public void testCacheOnWriteEvictOnClose() throws Exception { cacheConf = new CacheConfig(conf, bc); Path pathCowOn = new Path(baseDir, "123456788"); writer = writeStoreFile(conf, cacheConf, pathCowOn, 3); - hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); + storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile hsfCowOn = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); // Read this file, we should see 3 hits - hsf.initReader(); - reader = hsf.getReader(); + hsfCowOn.initReader(); + reader = hsfCowOn.getReader(); scanner = getStoreFileScanner(reader, true, true); scanner.seek(KeyValue.LOWESTKEY); while (scanner.next() != null) { @@ -1034,15 +1097,13 @@ public void testCacheOnWriteEvictOnClose() throws Exception { reader.close(cacheConf.shouldEvictOnClose()); // Let's read back the two files to ensure the blocks exactly match - hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true); - hsf.initReader(); - StoreFileReader readerOne = hsf.getReader(); + hsfCowOff.initReader(); + StoreFileReader readerOne = hsfCowOff.getReader(); readerOne.loadFileInfo(); StoreFileScanner scannerOne = getStoreFileScanner(readerOne, true, true); scannerOne.seek(KeyValue.LOWESTKEY); - hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true); - hsf.initReader(); - StoreFileReader readerTwo = hsf.getReader(); + hsfCowOn.initReader(); + StoreFileReader readerTwo = hsfCowOn.getReader(); readerTwo.loadFileInfo(); StoreFileScanner scannerTwo = getStoreFileScanner(readerTwo, true, true); scannerTwo.seek(KeyValue.LOWESTKEY); @@ -1071,9 +1132,8 @@ public void testCacheOnWriteEvictOnClose() throws Exception { // Let's close the first file with evict on close turned on conf.setBoolean("hbase.rs.evictblocksonclose", true); cacheConf = new CacheConfig(conf, bc); - hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true); - hsf.initReader(); - reader = hsf.getReader(); + hsfCowOff.initReader(); + reader = hsfCowOff.getReader(); reader.close(cacheConf.shouldEvictOnClose()); // We should have 3 new evictions but the evict count stat should not change. Eviction because @@ -1085,9 +1145,8 @@ public void testCacheOnWriteEvictOnClose() throws Exception { // Let's close the second file with evict on close turned off conf.setBoolean("hbase.rs.evictblocksonclose", false); cacheConf = new CacheConfig(conf, bc); - hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true); - hsf.initReader(); - reader = hsf.getReader(); + hsfCowOn.initReader(); + reader = hsfCowOn.getReader(); reader.close(cacheConf.shouldEvictOnClose()); // We expect no changes @@ -1097,9 +1156,9 @@ public void testCacheOnWriteEvictOnClose() throws Exception { } private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo hri, - final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef) - throws IOException { - Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null); + final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef, + StoreFileTracker sft) throws IOException { + Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null, sft); if (null == path) { return null; } @@ -1166,8 +1225,9 @@ public void testDataBlockEncodingMetaData() throws IOException { .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build(); writer.close(); - HStoreFile storeFile = - new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile storeFile = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); @@ -1195,8 +1255,9 @@ public void testDataBlockSizeEncoded() throws Exception { .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build(); writeStoreFile(writer); - HStoreFile storeFile = - new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile storeFile = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); @@ -1254,8 +1315,9 @@ private void testDataBlockSizeWithCompressionRatePredicator(int expectedBlockCou writeLargeStoreFile(writer, Bytes.toBytes(name.getMethodName()), Bytes.toBytes(name.getMethodName()), 200); writer.close(); - HStoreFile storeFile = - new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); + StoreFileInfo storeFileInfo = + StoreFileInfo.createStoreFileInfoForHFile(conf, fs, writer.getPath(), true); + HStoreFile storeFile = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); storeFile.initReader(); HFile.Reader fReader = HFile.createReader(fs, writer.getPath(), storeFile.getCacheConf(), true, conf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java index 84437335d83e..36597910172c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java @@ -47,6 +47,8 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -121,11 +123,16 @@ public void testCommitDaughterRegion() throws Exception { .setRegionId(region.getRegionInfo().getRegionId()).build(); HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; List splitFilesA = new ArrayList<>(); + HRegionFileSystem regionFs = region.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(region.getBaseConf(), true, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), "info")) + .withRegionFileSystem(regionFs).build()); splitFilesA.add(regionFS.splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), false, region.getSplitPolicy())); + Bytes.toBytes("002"), false, region.getSplitPolicy(), sft)); List splitFilesB = new ArrayList<>(); splitFilesB.add(regionFS.splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file, - Bytes.toBytes("002"), true, region.getSplitPolicy())); + Bytes.toBytes("002"), true, region.getSplitPolicy(), sft)); MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getEnvironment(); Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env); @@ -219,7 +226,13 @@ public void testMergeLoadsFromTracker() throws Exception { private Pair copyFileInTheStoreDir(HRegion region) throws IOException { Path storeDir = region.getRegionFileSystem().getStoreDir("info"); // gets the single file - StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0); + HRegionFileSystem regionFs = region.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(region.getBaseConf(), false, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(storeDir) + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME)) + .withRegionFileSystem(regionFs).build()); + List infos = sft.load(); + StoreFileInfo fileInfo = infos.get(0); // make a copy of the valid file staight into the store dir, so that it's not tracked. String copyName = UUID.randomUUID().toString().replaceAll("-", ""); Path copy = new Path(storeDir, copyName); @@ -231,7 +244,13 @@ private Pair copyFileInTheStoreDir(HRegion region) throws private void validateDaughterRegionsFiles(HRegion region, String originalFileName, String untrackedFile) throws IOException { // verify there's no link for the untracked, copied file in first region - List infos = region.getRegionFileSystem().getStoreFiles("info"); + HRegionFileSystem regionFs = region.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), false, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), "info")) + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME)) + .withRegionFileSystem(regionFs).build()); + List infos = sft.load(); assertThat(infos, everyItem(hasProperty("activeFileName", not(containsString(untrackedFile))))); assertThat(infos, hasItem(hasProperty("activeFileName", containsString(originalFileName)))); } @@ -246,7 +265,13 @@ private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Excepti private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS) throws IOException { HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0]; - return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file); + HRegionFileSystem regionFs = regionToMerge.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(regionToMerge.getBaseConf(), true, + StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), FAMILY_NAME_STR)) + .withRegionFileSystem(regionFs).build()); + return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file, + sft); } private void putThreeRowsAndFlush(TableName table) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 2f4bfbe7dbe8..549371f6cc37 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -60,6 +60,8 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -247,7 +249,9 @@ public void testCleanMergeReference() throws Exception { new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo); int count = 0; for (ColumnFamilyDescriptor colFamily : columnFamilies) { - count += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); + StoreFileTracker sft = StoreFileTrackerFactory.create(TEST_UTIL.getConfiguration(), + tableDescriptor, colFamily, hrfs, false); + count += sft.load().size(); } ADMIN.compactRegion(mergedRegionInfo.getRegionName()); // clean up the merged region store files @@ -256,7 +260,9 @@ public void testCleanMergeReference() throws Exception { int newcount = 0; while (EnvironmentEdgeManager.currentTime() < timeout) { for (ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); + StoreFileTracker sft = StoreFileTrackerFactory.create(TEST_UTIL.getConfiguration(), + tableDescriptor, colFamily, hrfs, false); + newcount += sft.load().size(); } if (newcount > count) { break; @@ -275,7 +281,9 @@ public void testCleanMergeReference() throws Exception { while (EnvironmentEdgeManager.currentTime() < timeout) { int newcount1 = 0; for (ColumnFamilyDescriptor colFamily : columnFamilies) { - newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size(); + StoreFileTracker sft = StoreFileTrackerFactory.create(TEST_UTIL.getConfiguration(), + tableDescriptor, colFamily, hrfs, false); + newcount1 += sft.load().size(); } if (newcount1 <= 1) { break; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index d56787aba115..df32897876c0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -121,8 +121,9 @@ public void testReversibleStoreFileScanner() throws IOException { .withOutputDir(hfilePath).withFileContext(hFileContext).build(); writeStoreFile(writer); - HStoreFile sf = new HStoreFile(fs, writer.getPath(), TEST_UTIL.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, writer.getPath(), true); + HStoreFile sf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); List scanners = StoreFileScanner.getScannersForStoreFiles( Collections.singletonList(sf), false, true, false, false, Long.MAX_VALUE); @@ -172,11 +173,13 @@ public void testReversibleKeyValueHeap() throws IOException { MemStore memstore = new DefaultMemStore(); writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1, writer2 }); - HStoreFile sf1 = new HStoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo1 = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, writer1.getPath(), true); + HStoreFile sf1 = new HStoreFile(storeFileInfo1, BloomType.NONE, cacheConf); - HStoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo2 = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, writer2.getPath(), true); + HStoreFile sf2 = new HStoreFile(storeFileInfo2, BloomType.NONE, cacheConf); /** * Test without MVCC */ @@ -252,11 +255,13 @@ public void testReversibleStoreScanner() throws IOException { MemStore memstore = new DefaultMemStore(); writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1, writer2 }); - HStoreFile sf1 = new HStoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo1 = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, writer1.getPath(), true); + HStoreFile sf1 = new HStoreFile(storeFileInfo1, BloomType.NONE, cacheConf); - HStoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf, - BloomType.NONE, true); + StoreFileInfo storeFileInfo2 = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, writer2.getPath(), true); + HStoreFile sf2 = new HStoreFile(storeFileInfo2, BloomType.NONE, cacheConf); ScanInfo scanInfo = new ScanInfo(TEST_UTIL.getConfiguration(), FAMILYNAME, 0, Integer.MAX_VALUE, Long.MAX_VALUE, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowPrefixBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowPrefixBloomFilter.java index acd5362e0363..b77fae0677d5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowPrefixBloomFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowPrefixBloomFilter.java @@ -186,7 +186,7 @@ public void testRowPrefixBloomFilter() throws Exception { // read the file ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -259,7 +259,7 @@ public void testRowPrefixBloomFilterWithGet() throws Exception { writeStoreFile(f, bt, expKeys); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); @@ -315,7 +315,7 @@ public void testRowPrefixBloomFilterWithScan() throws Exception { writeStoreFile(f, bt, expKeys); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 91bbea575309..db2a9d68f288 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -91,6 +91,8 @@ import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -951,11 +953,14 @@ public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck() Collection storefiles = store.getStorefiles(); assertEquals(1, storefiles.size()); assertFalse(region.hasReferences()); - Path referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); + HRegionFileSystem hfs = region.getRegionFileSystem(); + StoreFileTracker sft = StoreFileTrackerFactory.create(TESTING_UTIL.getConfiguration(), true, + store.getStoreContext()); + Path referencePath = hfs.splitStoreFile(region.getRegionInfo(), "f", + storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft); assertNull(referencePath); - referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f", - storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy()); + referencePath = hfs.splitStoreFile(region.getRegionInfo(), "i_f", + storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy(), sft); assertNotNull(referencePath); } finally { TESTING_UTIL.deleteTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java index e7e385d9ffbf..0aa47048945f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java @@ -28,10 +28,15 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.ClassRule; @@ -97,23 +102,6 @@ public void testEqualsWithLink() throws IOException { assertEquals(info1.hashCode(), info2.hashCode()); } - @Test - public void testOpenErrorMessageHFileLink() throws IOException, IllegalStateException { - // Test file link exception - // Try to open nonsense hfilelink. Make sure exception is from HFileLink. - Path p = new Path("/hbase/test/0123/cf/testtb=4567-abcd"); - try (FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration())) { - StoreFileInfo sfi = new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, p, true); - try { - ReaderContext context = sfi.createReaderContext(false, 1000, ReaderType.PREAD); - sfi.createReader(context, null); - throw new IllegalStateException(); - } catch (FileNotFoundException fnfe) { - assertTrue(fnfe.getMessage().contains(HFileLink.class.getSimpleName())); - } - } - } - @Test public void testOpenErrorMessageReference() throws IOException { // Test file link exception @@ -122,8 +110,17 @@ public void testOpenErrorMessageReference() throws IOException { FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); fs.mkdirs(p.getParent()); Reference r = Reference.createBottomReference(HConstants.EMPTY_START_ROW); - r.write(fs, p); - StoreFileInfo sfi = new StoreFileInfo(TEST_UTIL.getConfiguration(), fs, p, true); + RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf("table1")).build(); + StoreContext storeContext = StoreContext.getBuilder() + .withRegionFileSystem(HRegionFileSystem.create(TEST_UTIL.getConfiguration(), fs, + TEST_UTIL.getDataTestDirOnTestFS(), regionInfo)) + .withColumnFamilyDescriptor( + ColumnFamilyDescriptorBuilder.newBuilder("cf1".getBytes()).build()) + .build(); + StoreFileTrackerForTest storeFileTrackerForTest = + new StoreFileTrackerForTest(TEST_UTIL.getConfiguration(), true, storeContext); + storeFileTrackerForTest.createReference(r, p); + StoreFileInfo sfi = storeFileTrackerForTest.getStoreFileInfo(p, true); try { ReaderContext context = sfi.createReaderContext(false, 1000, ReaderType.PREAD); sfi.createReader(context, null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 79e2f797dc9f..5f36d201a753 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -46,6 +47,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.storefiletracker.FailingStoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -80,41 +82,36 @@ public void setUp() throws IOException { } private TableDescriptor getTableDesc(TableName tableName, int regionReplication, - byte[]... families) { - return getTableDesc(tableName, regionReplication, false, families); + String trackerName, byte[]... families) { + return getTableDesc(tableName, regionReplication, false, trackerName, families); } private TableDescriptor getTableDesc(TableName tableName, int regionReplication, boolean readOnly, - byte[]... families) { + String trackerName, byte[]... families) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName) .setRegionReplication(regionReplication).setReadOnly(readOnly); + if (trackerName != null) { + builder.setValue(TRACKER_IMPL, trackerName); + } Arrays.stream(families).map(family -> ColumnFamilyDescriptorBuilder.newBuilder(family) .setMaxVersions(Integer.MAX_VALUE).build()).forEachOrdered(builder::setColumnFamily); return builder.build(); } - static class FailingHRegionFileSystem extends HRegionFileSystem { - boolean fail = false; + public static class FailingHRegionFileSystem extends HRegionFileSystem { + public boolean fail = false; FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, RegionInfo regionInfo) { super(conf, fs, tableDir, regionInfo); } - @Override - public List getStoreFiles(String familyName) throws IOException { - if (fail) { - throw new IOException("simulating FS failure"); - } - return super.getStoreFiles(familyName); - } } private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName()); - RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey) .setEndKey(stopKey).setRegionId(0L).setReplicaId(replicaId).build(); HRegionFileSystem fs = @@ -200,7 +197,9 @@ public void testIsStale() throws IOException { when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions); when(regionServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); - TableDescriptor htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, families); + String trackerName = FailingStoreFileTrackerForTest.class.getName(); + TableDescriptor htd = + getTableDesc(TableName.valueOf(name.getMethodName()), 2, trackerName, families); HRegion primary = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 0); HRegion replica1 = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 1); regions.add(primary); @@ -252,7 +251,7 @@ public void testRefreshReadOnlyTable() throws IOException { when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions); when(regionServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); - TableDescriptor htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, families); + TableDescriptor htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, null, families); HRegion primary = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 0); HRegion replica1 = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 1); regions.add(primary); @@ -276,11 +275,12 @@ public void testRefreshReadOnlyTable() throws IOException { verifyData(primary, 0, 200, qf, families); // then the table is set to readonly - htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, true, families); + htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, true, null, families); primary.setTableDescriptor(htd); replica1.setTableDescriptor(htd); chore.chore(); // we cannot refresh the store files verifyDataExpectFail(replica1, 100, 100, qf, families); } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileScannerWithTagCompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileScannerWithTagCompression.java index 6a251539ccba..9dd8271d7bc0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileScannerWithTagCompression.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileScannerWithTagCompression.java @@ -84,7 +84,7 @@ public void testReseek() throws Exception { writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(conf, fs, f, true); storeFileInfo.initHFileInfo(context); StoreFileReader reader = storeFileInfo.createReader(context, cacheConf); storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java index 2f2bc0033d7b..c0b7621e9eb7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java @@ -128,7 +128,8 @@ public void testScannerCloseAndUpdateReadersWithMemstoreScanner() throws Excepti region.put(p); HStore store = region.getStore(fam); // use the lock to manually get a new memstore scanner. this is what - // HStore#notifyChangedReadersObservers does under the lock.(lock is not needed here + // HStore#notifyChangedReadersObservers does under the lock.(lock is not needed + // here // since it is just a testcase). store.getStoreEngine().readLock(); final List memScanners = store.memstore.getScanners(Long.MAX_VALUE); @@ -213,9 +214,9 @@ private static KeyValue.Type generateKeyType(Random rand) { } } - private HStoreFile readStoreFile(Path storeFilePath, Configuration conf) throws Exception { + private HStoreFile readStoreFile(StoreFileInfo fileinfo) throws Exception { // Open the file reader with block cache disabled. - HStoreFile file = new HStoreFile(fs, storeFilePath, conf, cacheConf, BloomType.NONE, true); + HStoreFile file = new HStoreFile(fileinfo, BloomType.NONE, cacheConf); return file; } @@ -226,7 +227,8 @@ private void testScannerCloseAndUpdateReaderInternal(boolean awaitUpdate, boolea HStoreFile file = null; List files = new ArrayList(); try { - file = readStoreFile(path, CONF); + StoreFileInfo storeFileInfo = StoreFileInfo.createStoreFileInfoForHFile(CONF, fs, path, true); + file = readStoreFile(storeFileInfo); files.add(file); } catch (Exception e) { // fail test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java index 2037b738e433..a479550d7e69 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java @@ -574,7 +574,10 @@ private static MockHStoreFile createFile(long size, long seqNum, byte[] startKey FileSystem fs = TEST_UTIL.getTestFileSystem(); Path testFilePath = StoreFileWriter.getUniqueFile(fs, CFDIR); fs.create(testFilePath).close(); - MockHStoreFile sf = new MockHStoreFile(TEST_UTIL, testFilePath, size, 0, false, seqNum); + StoreFileInfo storeFileInfo = StoreFileInfo + .createStoreFileInfoForHFile(TEST_UTIL.getConfiguration(), fs, testFilePath, true); + MockHStoreFile sf = + new MockHStoreFile(TEST_UTIL, testFilePath, size, 0, false, seqNum, storeFileInfo); if (startKey != null) { sf.setMetadataValue(StripeStoreFileManager.STRIPE_START_KEY, startKey); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FailingStoreFileTrackerForTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FailingStoreFileTrackerForTest.java new file mode 100644 index 000000000000..34a279db5b61 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FailingStoreFileTrackerForTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.storefiletracker; + +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.TestStoreFileRefresherChore.FailingHRegionFileSystem; + +public class FailingStoreFileTrackerForTest extends DefaultStoreFileTracker { + + FailingStoreFileTrackerForTest(Configuration conf, boolean isPrimaryReplica, StoreContext ctx) { + super(conf, isPrimaryReplica, ctx); + } + + @Override + protected List doLoadStoreFiles(boolean readOnly) throws IOException { + if (ctx.getRegionFileSystem() instanceof FailingHRegionFileSystem) { + if (((FailingHRegionFileSystem) ctx.getRegionFileSystem()).fail) { + throw new IOException("simulating FS failure"); + } + } + return super.doLoadStoreFiles(readOnly); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerForTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerForTest.java index a6ab40b59d82..c2fe9afc7020 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerForTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerForTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.slf4j.Logger; @@ -69,4 +70,10 @@ public static boolean tracked(String encodedRegionName, String family, Path file public static void clear() { trackedFiles.clear(); } + + @Override + public Reference readReference(Path p) throws IOException { + return super.readReference(p); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java index 02b122f704a7..af1f8ad561d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java @@ -31,11 +31,15 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -111,7 +115,10 @@ public void testIsStoreFileSizeMatchFilesystemAndManifest() throws IOException { for (RegionInfo regionInfo : regionsInfo) { HRegionFileSystem hRegionFileSystem = HRegionFileSystem.openRegionFromFileSystem(conf, fs, path, regionInfo, true); - Collection storeFilesFS = hRegionFileSystem.getStoreFiles(FAMILY_NAME); + ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(FAMILY_NAME); + StoreFileTracker sft = + StoreFileTrackerFactory.create(conf, table.getDescriptor(), hcd, hRegionFileSystem); + Collection storeFilesFS = sft.load(); Iterator sfIterator = storeFilesFS.iterator(); while (sfIterator.hasNext()) { StoreFileInfo sfi = sfIterator.next(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java index 981e312043ea..962f825ffece 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Before; @@ -95,8 +96,8 @@ public void testStoresNeedingCompaction() throws Exception { public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception { // this tests that reference files that are new, but have older timestamps for the files // they reference still will get compacted. - TableName table = TableName.valueOf("TestMajorCompactor"); - TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY)); + TableName tableName = TableName.valueOf("TestMajorCompactor"); + TableDescriptor htd = UTILITY.createTableDescriptor(tableName, Bytes.toBytes(FAMILY)); RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd); @@ -111,12 +112,23 @@ public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception { spy(new MajorCompactionRequest(connection, region.getRegionInfo(), Sets.newHashSet(FAMILY))); doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class), any(Path.class)); + StoreFileTrackerForTest sft = mockSFT(true, storeFiles); doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(); + doReturn(sft).when(majorCompactionRequest).getStoreFileTracker(any(), any()); + doReturn(UTILITY.getConfiguration()).when(connection).getConfiguration(); Set result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100); assertEquals(FAMILY, Iterables.getOnlyElement(result)); } + protected StoreFileTrackerForTest mockSFT(boolean references, List storeFiles) + throws IOException { + StoreFileTrackerForTest sft = mock(StoreFileTrackerForTest.class); + doReturn(references).when(sft).hasReferences(); + doReturn(storeFiles).when(sft).load(); + return sft; + } + protected HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, List storeFiles) throws IOException { long timestamp = storeFiles.stream().findFirst().get().getModificationTime(); @@ -135,7 +147,6 @@ private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFi doReturn(info).when(mockSystem).getRegionInfo(); doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY); doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString()); - doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString()); doReturn(fileSystem).when(mockSystem).getFileSystem(); return mockSystem; } @@ -165,6 +176,8 @@ private MajorCompactionRequest makeMockRequest(List storeFiles, b new MajorCompactionRequest(connection, regionInfo, Sets.newHashSet("a")); MajorCompactionRequest spy = spy(request); HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles); + StoreFileTrackerForTest sft = mockSFT(references, storeFiles); + doReturn(sft).when(spy).getStoreFileTracker(any(), any()); doReturn(fileSystem).when(spy).getFileSystem(); return spy; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionTTLRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionTTLRequest.java index f941282039f0..b3fab4c0b5c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionTTLRequest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionTTLRequest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Before; import org.junit.ClassRule; @@ -89,6 +91,8 @@ private MajorCompactionTTLRequest makeMockRequest(List storeFiles MajorCompactionTTLRequest request = new MajorCompactionTTLRequest(connection, regionInfo); MajorCompactionTTLRequest spy = spy(request); HRegionFileSystem fileSystem = mockFileSystem(regionInfo, false, storeFiles); + StoreFileTrackerForTest sft = mockSFT(false, storeFiles); + doReturn(sft).when(spy).getStoreFileTracker(any(), any()); doReturn(fileSystem).when(spy).getFileSystem(); return spy; } From 6290079d2055561634d02bb3cf20f972a59633c4 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 21 Oct 2025 17:01:48 -0700 Subject: [PATCH 089/336] Preparing hbase release 2.6.4RC0; tagging and updates to CHANGES.md and RELEASENOTES.md Signed-off-by: Andrew Purtell --- CHANGES.md | 117 ++++++++++++++++++++++++++++++++++++++++++++++++ RELEASENOTES.md | 48 ++++++++++++++++++++ pom.xml | 2 +- 3 files changed, 166 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index debb62e9a2ad..47dbc41ee716 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,123 @@ --> # HBASE Changelog +## Release 2.6.4 - Unreleased (as of 2025-10-22) + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-28463](https://issues.apache.org/jira/browse/HBASE-28463) | Time Based Priority for BucketCache | Major | BucketCache | +| [HBASE-28919](https://issues.apache.org/jira/browse/HBASE-28919) | Soft drop for destructive table actions | Major | master, snapshots | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29663](https://issues.apache.org/jira/browse/HBASE-29663) | TimeBasedLimiters should support dynamic configuration refresh | Major | . | +| [HBASE-29653](https://issues.apache.org/jira/browse/HBASE-29653) | Build fails on riscv64 due to os-maven-plugin not recognizing RISC-V architecture | Major | build | +| [HBASE-29650](https://issues.apache.org/jira/browse/HBASE-29650) | Upgrade tomcat-jasper to 9.0.110 | Major | UI | +| [HBASE-29637](https://issues.apache.org/jira/browse/HBASE-29637) | Implement ResourceCheckerJUnitListener for junit 5 | Major | test | +| [HBASE-29649](https://issues.apache.org/jira/browse/HBASE-29649) | Un-deprecate preWALRestore and postWALRestore in RegionCoprocessorHost | Minor | Coprocessors | +| [HBASE-29636](https://issues.apache.org/jira/browse/HBASE-29636) | Implement TimedOutTestsListener for junit 5 | Major | test | +| [HBASE-29626](https://issues.apache.org/jira/browse/HBASE-29626) | Refactor server side scan metrics for Coproc hooks | Minor | . | +| [HBASE-28440](https://issues.apache.org/jira/browse/HBASE-28440) | Add support for using mapreduce sort in HFileOutputFormat2 | Major | backup&restore | +| [HBASE-29627](https://issues.apache.org/jira/browse/HBASE-29627) | Handle any block cache fetching errors when reading a block in HFileReaderImpl | Major | BlockCache | +| [HBASE-29576](https://issues.apache.org/jira/browse/HBASE-29576) | Replicate HBaseClassTestRule functionality for Junit 5 | Major | test | +| [HBASE-29612](https://issues.apache.org/jira/browse/HBASE-29612) | Remove HBaseTestingUtil.forceChangeTaskLogDir | Major | . | +| [HBASE-29608](https://issues.apache.org/jira/browse/HBASE-29608) | Add test to make sure we do not have copy paste errors in the TAG value | Minor | test | +| [HBASE-29610](https://issues.apache.org/jira/browse/HBASE-29610) | Add and use String constants for Junit 5 @Tag annotations | Minor | integration tests, test | +| [HBASE-29573](https://issues.apache.org/jira/browse/HBASE-29573) | Fully load QuotaCache instead of reading individual rows on demand | Minor | . | +| [HBASE-29571](https://issues.apache.org/jira/browse/HBASE-29571) | Fix Javadoc typo: 'repoen' should be 'reopen' | Trivial | . | +| [HBASE-29575](https://issues.apache.org/jira/browse/HBASE-29575) | Do not limit surefire to Junit 4 | Major | test | +| [HBASE-29496](https://issues.apache.org/jira/browse/HBASE-29496) | Fix Javadoc typo: 'DsiableTableProcedure' should be 'DisableTableProcedure' | Trivial | documentation | +| [HBASE-29494](https://issues.apache.org/jira/browse/HBASE-29494) | Capture Scan RPC processing time and queuing time in Scan Metrics | Minor | . | +| [HBASE-29479](https://issues.apache.org/jira/browse/HBASE-29479) | QuotaCache is not correctly populated until runs of QuotaRefresherChore | Minor | . | +| [HBASE-29556](https://issues.apache.org/jira/browse/HBASE-29556) | Display HBCK and CatalogJanitor report errors properly on HBCK Report page | Major | UI | +| [HBASE-29431](https://issues.apache.org/jira/browse/HBASE-29431) | Update the 'ExcludeDNs' information with the cause in RS UI | Major | UI | +| [HBASE-29473](https://issues.apache.org/jira/browse/HBASE-29473) | Obtain target cluster's token for cross clusters job | Major | . | +| [HBASE-29528](https://issues.apache.org/jira/browse/HBASE-29528) | Support for cellVisibility in Thrift interface | Minor | Thrift | +| [HBASE-29290](https://issues.apache.org/jira/browse/HBASE-29290) | Include port number of Region Server in the Replication Status message | Minor | shell | +| [HBASE-29469](https://issues.apache.org/jira/browse/HBASE-29469) | Add RPC throttling metrics to RegionServer for quota monitoring | Minor | metrics | +| [HBASE-29508](https://issues.apache.org/jira/browse/HBASE-29508) | Define HBase specific TLS config properties for InfoServer | Major | . | +| [HBASE-29477](https://issues.apache.org/jira/browse/HBASE-29477) | Make TableOutputCommitter Configurable for TableOutputFormat | Blocker | . | +| [HBASE-29481](https://issues.apache.org/jira/browse/HBASE-29481) | Make TLS protocols and include cipher list configurable for HTTPS InfoServer | Major | security, UI | +| [HBASE-15625](https://issues.apache.org/jira/browse/HBASE-15625) | Make minimum values configurable and smaller | Minor | . | +| [HBASE-29467](https://issues.apache.org/jira/browse/HBASE-29467) | Redundant conditions in CostFunction.scale() method | Major | Balancer | +| [HBASE-29450](https://issues.apache.org/jira/browse/HBASE-29450) | Bump org.apache.commons:commons-lang3 from 3.17.0 to 3.18.0 | Major | dependabot, dependencies, security | +| [HBASE-29398](https://issues.apache.org/jira/browse/HBASE-29398) | Server side scan metrics for bytes read from FS vs Block cache vs memstore | Major | . | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29604](https://issues.apache.org/jira/browse/HBASE-29604) | BackupHFileCleaner uses flawed time based check | Critical | backup&restore | +| [HBASE-29629](https://issues.apache.org/jira/browse/HBASE-29629) | Record the quota user name value on metrics for RpcThrottlingExceptions | Minor | Quotas | +| [HBASE-29623](https://issues.apache.org/jira/browse/HBASE-29623) | Blocks for CFs with BlockCache disabled may still get cached on write or compaction | Major | BlockCache | +| [HBASE-29550](https://issues.apache.org/jira/browse/HBASE-29550) | Reflection error in TestRSGroupsKillRS with Java 21 | Major | test | +| [HBASE-29587](https://issues.apache.org/jira/browse/HBASE-29587) | Set Test category for TestSnapshotProcedureEarlyExpiration | Minor | snapshots | +| [HBASE-29601](https://issues.apache.org/jira/browse/HBASE-29601) | Handle Junit 5 tests in TestCheckTestClasses | Major | test | +| [HBASE-29602](https://issues.apache.org/jira/browse/HBASE-29602) | Add -Djava.security.manager=allow to JDK18+ surefire JVM flags | Major | integration tests, test | +| [HBASE-29548](https://issues.apache.org/jira/browse/HBASE-29548) | Update ApacheDS to 2.0.0.AM27 and ldap-api to 2.1.7 | Major | test | +| [HBASE-29577](https://issues.apache.org/jira/browse/HBASE-29577) | Fix NPE from RegionServerRpcQuotaManager when reloading configuration | Minor | . | +| [HBASE-27157](https://issues.apache.org/jira/browse/HBASE-27157) | Potential race condition in WorkerAssigner | Minor | proc-v2 | +| [HBASE-29566](https://issues.apache.org/jira/browse/HBASE-29566) | TestPrefetch.testPrefetchWithDelay seems flakey | Major | . | +| [HBASE-29453](https://issues.apache.org/jira/browse/HBASE-29453) | NPE on CacheAwareLoadBalancer.balanceTable | Trivial | Balancer | +| [HBASE-29540](https://issues.apache.org/jira/browse/HBASE-29540) | Unhandled IllegalArgumentException in HBase Web UI When Accessing table.jsp with Invalid Table Name | Minor | UI | +| [HBASE-29558](https://issues.apache.org/jira/browse/HBASE-29558) | Broken hbase-shell no-cluster tests | Major | shell | +| [HBASE-29570](https://issues.apache.org/jira/browse/HBASE-29570) | Set no watches on the node when recursively deleting the node and its child nodes | Minor | Zookeeper | +| [HBASE-28881](https://issues.apache.org/jira/browse/HBASE-28881) | Setting \`hbase.master.procedure.threads\` to negative value doesn't break HMaster but clients cannot connect | Critical | master | +| [HBASE-29549](https://issues.apache.org/jira/browse/HBASE-29549) | Mockito failures in TestServerCall with Java 21 | Major | test | +| [HBASE-28866](https://issues.apache.org/jira/browse/HBASE-28866) | Setting \`hbase.oldwals.cleaner.thread.size\` to negative value will break HMaster and produce hard-to-diagnose logs | Critical | master | +| [HBASE-29532](https://issues.apache.org/jira/browse/HBASE-29532) | NPE error when there is EOF for specific recover folder | Major | . | +| [HBASE-29502](https://issues.apache.org/jira/browse/HBASE-29502) | RegionReplicaReplicationEndpoint fails to forward mutations when meta cache does not contain secondary replica locations | Major | read replicas | +| [HBASE-29544](https://issues.apache.org/jira/browse/HBASE-29544) | Assertion errors in BackupAndRestoreThread are not causing IntegrationTestBackupRestore to fail | Major | integration tests | +| [HBASE-29184](https://issues.apache.org/jira/browse/HBASE-29184) | The snapshot type for disabled table is incorrect when snapshot procedure is enabled | Major | snapshots | +| [HBASE-29493](https://issues.apache.org/jira/browse/HBASE-29493) | TestBucketCacheRefCnt.testInBucketCache fails if block found after eviction | Minor | BucketCache | +| [HBASE-29543](https://issues.apache.org/jira/browse/HBASE-29543) | TestFileChangeWatcher fails with Java 8 due to file modification time granularity issue | Minor | . | +| [HBASE-29507](https://issues.apache.org/jira/browse/HBASE-29507) | IntegrationTestBackupRestore is failing because it cannot restore from backup directory | Major | . | +| [HBASE-28951](https://issues.apache.org/jira/browse/HBASE-28951) | Handle simultaneous WAL splitting to recovered edits by multiple worker | Major | . | +| [HBASE-29503](https://issues.apache.org/jira/browse/HBASE-29503) | IntegrationTestBackupRestore is passing even if an exception occurs in the thread(s) it creates | Major | integration tests | +| [HBASE-29296](https://issues.apache.org/jira/browse/HBASE-29296) | Missing critical snapshot expiration checks | Critical | backup&restore, snapshots | +| [HBASE-29463](https://issues.apache.org/jira/browse/HBASE-29463) | Bidirectional serial replication will block if a region’s last edit before rs crashed was from the peer cluster | Critical | Replication | +| [HBASE-29482](https://issues.apache.org/jira/browse/HBASE-29482) | Bulkload fails with viewfs authentication error | Minor | . | +| [HBASE-29472](https://issues.apache.org/jira/browse/HBASE-29472) | Fix splitting algorithms of RegionSplitter tool | Minor | util | +| [HBASE-29447](https://issues.apache.org/jira/browse/HBASE-29447) | WAL Archives Cause Incremental Backup Failures | Major | backup&restore | +| [HBASE-29474](https://issues.apache.org/jira/browse/HBASE-29474) | RegionSplitter.rollingSplit is broken | Major | . | +| [HBASE-28589](https://issues.apache.org/jira/browse/HBASE-28589) | Server side DoNotRetryException not propagated to client | Critical | IPC/RPC | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-29614](https://issues.apache.org/jira/browse/HBASE-29614) | Remove static final field modification in tests around Unsafe | Major | test | +| [HBASE-29591](https://issues.apache.org/jira/browse/HBASE-29591) | Add hadoop 3.4.2 in hadoop check | Major | hadoop3, jenkins, scripts | +| [HBASE-29592](https://issues.apache.org/jira/browse/HBASE-29592) | Add hadoop 3.4.2 in client integration tests | Major | hadoop3, jenkins, scripts | +| [HBASE-29590](https://issues.apache.org/jira/browse/HBASE-29590) | Use hadoop 3.4.2 as default hadooop3 dependency | Major | dependencies, hadoop3 | +| [HBASE-29427](https://issues.apache.org/jira/browse/HBASE-29427) | Merge all commits related to custom tiering into the feature branch | Major | . | +| [HBASE-28467](https://issues.apache.org/jira/browse/HBASE-28467) | Integration of time-based priority caching into cacheOnRead read code paths. | Major | BucketCache | +| [HBASE-28469](https://issues.apache.org/jira/browse/HBASE-28469) | Integration of time-based priority caching into compaction paths. | Major | . | +| [HBASE-28535](https://issues.apache.org/jira/browse/HBASE-28535) | Implement a region server level configuration to enable/disable data-tiering | Major | BucketCache | +| [HBASE-28468](https://issues.apache.org/jira/browse/HBASE-28468) | Integration of time-based priority caching logic into cache evictions. | Major | . | +| [HBASE-28466](https://issues.apache.org/jira/browse/HBASE-28466) | Integration of time-based priority logic of bucket cache in prefetch functionality of HBase. | Major | BucketCache | +| [HBASE-28505](https://issues.apache.org/jira/browse/HBASE-28505) | Implement enforcement to require Date Tiered Compaction for Time Range Data Tiering | Major | . | +| [HBASE-28465](https://issues.apache.org/jira/browse/HBASE-28465) | Implementation of framework for time-based priority bucket-cache. | Major | . | + + +### OTHER: + +| JIRA | Summary | Priority | Component | +|:---- |:---- | :--- |:---- | +| [HBASE-23671](https://issues.apache.org/jira/browse/HBASE-23671) | Upgrade to JUnit 5 | Major | Filesystem Integration, test | +| [HBASE-29509](https://issues.apache.org/jira/browse/HBASE-29509) | Bump hbase-thirdparty to 4.1.12 | Major | dependencies, thirdparty | +| [HBASE-29527](https://issues.apache.org/jira/browse/HBASE-29527) | Bump org.bouncycastle:bcpkix-jdk18on from 1.78 to 1.81 | Major | dependabot, dependencies, security | + + ## Release 2.6.3 - Unreleased (as of 2025-07-10) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6e062b7d1876..8109d1783ba7 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,6 +16,54 @@ # See the License for the specific language governing permissions and # limitations under the License. --> +# HBASE 2.6.4 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HBASE-29663](https://issues.apache.org/jira/browse/HBASE-29663) | *Major* | **TimeBasedLimiters should support dynamic configuration refresh** + +You can now refresh TimeBasedRateLimiter configurations dynamically with a call to the HBase shell's update\_all\_configs + + +--- + +* [HBASE-29573](https://issues.apache.org/jira/browse/HBASE-29573) | *Minor* | **Fully load QuotaCache instead of reading individual rows on demand** + +Each RegionServer now always caches the full contents of the hbase:quota table. If you have a very large quota table, this could cause excessive memory usage. + + +--- + +* [HBASE-28463](https://issues.apache.org/jira/browse/HBASE-28463) | *Major* | **Time Based Priority for BucketCache** + +This introduces time based priority for blocks in the BucketCache. It's disabled by default. Allows for defining an age threshold at individual column family configuration, whereby blocks older than this configured threshold would be targeted first for eviction. Blocks from column families that don't define the age threshold wouldn't be evaluated by the time based priority, and would only be evicted following the pre-existing LRU eviction logic. + +To enable it, first set the hbase.regionserver.datatiering.enable property to true in the RegionServer configuration. Then, for each table column family where time based priority behaviour is desired, add the following properties to the related column families configurations: +- hbase.hstore.datatiering.type -\> TIME\_RANGE or CUSTOM +- hbase.hstore.datatiering.hot.age.millis -\> A milliseconds age value (defaults to 7 Days or 604800000 milliseconds) +- hbase.hstore.engine.class -\> org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine or org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine + +The TIME\_RANGE value for hbase.hstore.datatiering.type will rely on cells timestamps for calculating the block age to be compared against the hbase.hstore.datatiering.hot.age.millis threshold age to decide on the block priority. This option requires that org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine be defined as the hbase.hstore.engine.class. This is to enable date tiered compaction, so that data can be placed at separate files, according to the cells timestamps and the age threshold. + +The CUSTOM value for hbase.hstore.datatiering.type allows for defining custom logic to identify the age of cells that should be compared against the threshold age defined in the hbase.hstore.datatiering.hot.age.millis property. This option requires that org.apache.hadoop.hbase.regionserver.CustomTieredStoreEngine be defined as the hbase.hstore.engine.class. This is to enable the custom tiered compaction, so that data can be placed at separate files, according to the custom logic for defining the cell age to be compared against the age threshold. The custom logic for defining cell age should be provided as implementations of the CustomTieredCompactor.TieringValueProvider interface, and should be specified as the value of the hbase.hstore.custom-tiering-value.provider.class. + +Additionally, a built-in implementation of CustomTieredCompactor.TieringValueProvider is provided and set by default when the CUSTOM value for hbase.hstore.datatiering.type is in use. This assumes a custom column qualifier value to contain a long timestamp to be used as the cell age to be compared against the configured age threshold. This column qualifier should be configured as the TIERING\_CELL\_QUALIFIER property in the given column family configuration. + +Note that major compaction needs to be completed on the related tables once the feature is configured properly at the related column families configurations. + + +--- + +* [HBASE-15625](https://issues.apache.org/jira/browse/HBASE-15625) | *Minor* | **Make minimum values configurable and smaller** + +Introduced a new configuration \`hbase.regionserver.free.heap.min.memory.size\`. +This configuration allows users to specify the minimum required amount of free heap memory using a human-readable format (e.g., 512m, 4g). By default, it remains consistent with the previous behavior, reserving 20% of the total heap size as free memory. This new option helps modern deployments with large heap sizes fine-tune memory usage more aggressively for MemStore and block cache configurations. + + + # HBASE 2.6.3 Release Notes These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. diff --git a/pom.xml b/pom.xml index 8d048c479fd8..68409106bb54 100644 --- a/pom.xml +++ b/pom.xml @@ -523,7 +523,7 @@ - 2.6.4-SNAPSHOT + 2.6.4 false From 6f3e29ca13c536f1881ea12d60fb525949c56a10 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 21 Oct 2025 17:02:10 -0700 Subject: [PATCH 090/336] Preparing development version 2.6.5-SNAPSHOT Signed-off-by: Andrew Purtell --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 68409106bb54..370173515e8d 100644 --- a/pom.xml +++ b/pom.xml @@ -523,7 +523,7 @@ - 2.6.4 + 2.6.5-SNAPSHOT false From 2ab16b8b52daa511bb60db4216e454e17b29b535 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Wed, 22 Oct 2025 10:00:26 -0400 Subject: [PATCH 091/336] HBASE-29677: Thread safety in QuotaRefresherChore (#7401) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/QuotaCache.java | 79 ++++++++++--------- .../hadoop/hbase/quotas/TestQuotaCache2.java | 46 +++++++++++ 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index c95578dc5d00..34104752e81d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.time.Duration; import java.util.EnumSet; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -70,10 +69,10 @@ public class QuotaCache implements Stoppable { private final Object initializerLock = new Object(); private volatile boolean initialized = false; - private volatile Map namespaceQuotaCache = new HashMap<>(); - private volatile Map tableQuotaCache = new HashMap<>(); - private volatile Map userQuotaCache = new HashMap<>(); - private volatile Map regionServerQuotaCache = new HashMap<>(); + private volatile Map namespaceQuotaCache = new ConcurrentHashMap<>(); + private volatile Map tableQuotaCache = new ConcurrentHashMap<>(); + private volatile Map userQuotaCache = new ConcurrentHashMap<>(); + private volatile Map regionServerQuotaCache = new ConcurrentHashMap<>(); private volatile boolean exceedThrottleQuotaEnabled = false; // factors used to divide cluster scope quota into machine scope quota @@ -310,44 +309,48 @@ public synchronized boolean triggerNow() { @Override protected void chore() { - updateQuotaFactors(); + synchronized (this) { + LOG.info("Reloading quota cache from hbase:quota table"); + updateQuotaFactors(); + + try { + Map newUserQuotaCache = + new ConcurrentHashMap<>(fetchUserQuotaStateEntries()); + updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); + userQuotaCache = newUserQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching user quotas", e); + } - try { - Map newUserQuotaCache = new HashMap<>(fetchUserQuotaStateEntries()); - updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); - userQuotaCache = newUserQuotaCache; - } catch (IOException e) { - LOG.error("Error while fetching user quotas", e); - } + try { + Map newRegionServerQuotaCache = + new ConcurrentHashMap<>(fetchRegionServerQuotaStateEntries()); + updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); + regionServerQuotaCache = newRegionServerQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching region server quotas", e); + } - try { - Map newRegionServerQuotaCache = - new HashMap<>(fetchRegionServerQuotaStateEntries()); - updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); - regionServerQuotaCache = newRegionServerQuotaCache; - } catch (IOException e) { - LOG.error("Error while fetching region server quotas", e); - } + try { + Map newTableQuotaCache = + new ConcurrentHashMap<>(fetchTableQuotaStateEntries()); + updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); + tableQuotaCache = newTableQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing table quotas", e); + } - try { - Map newTableQuotaCache = - new HashMap<>(fetchTableQuotaStateEntries()); - updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); - tableQuotaCache = newTableQuotaCache; - } catch (IOException e) { - LOG.error("Error while refreshing table quotas", e); - } + try { + Map newNamespaceQuotaCache = + new ConcurrentHashMap<>(fetchNamespaceQuotaStateEntries()); + updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); + namespaceQuotaCache = newNamespaceQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing namespace quotas", e); + } - try { - Map newNamespaceQuotaCache = - new HashMap<>(fetchNamespaceQuotaStateEntries()); - updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); - namespaceQuotaCache = newNamespaceQuotaCache; - } catch (IOException e) { - LOG.error("Error while refreshing namespace quotas", e); + fetchExceedThrottleQuota(); } - - fetchExceedThrottleQuota(); } private void fetchExceedThrottleQuota() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java index 8f8ac4991ca6..3e829b5c08af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -131,4 +131,50 @@ public void testForgetsDeletedQuota() { assertTrue(newCache.containsKey("my_table2")); assertFalse(newCache.containsKey("my_table1")); } + + @Test + public void testLearnsNewQuota() { + Map oldCache = new HashMap<>(); + + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table1", newState); + + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + assertTrue(newCache.containsKey("my_table1")); + } + + @Test + public void testUserSpecificOverridesDefaultNewQuota() { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); + oldState.setGlobalLimiter(limiter1); + + // establish new cache, with a limiter for 999 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(999).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 999 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(999, updatedLimiter.getReadAvailable()); + } } From d698d4040e56e2f7eb2e9d2ba37fdae5910c802f Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 28 Oct 2025 09:10:44 -0400 Subject: [PATCH 092/336] HBASE-29679: Suppress stack trace in RpcThrottlingException (#7403) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/RpcThrottlingException.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java index d4ab38f5bf73..b08179a27a58 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java @@ -205,4 +205,15 @@ protected static long timeFromString(String timeDiff) { } return -1; } + + /** + * There is little value in an RpcThrottlingException having a stack trace, since its cause is + * well understood without one. When a RegionServer is under heavy load and needs to serve many + * RpcThrottlingExceptions, skipping fillInStackTrace() will save CPU time and allocations, both + * here and later when the exception must be serialized over the wire. + */ + @Override + public synchronized Throwable fillInStackTrace() { + return this; + } } From 3bed95feb7d218fbca506f192b3271ab7f663aed Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 28 Oct 2025 12:05:39 -0700 Subject: [PATCH 093/336] Revert "HBASE-29473 Obtain target cluster's token for cross clusters job (#7198)" Compatibility issue, see JIRA This reverts commit 22ba3bf98d132e828c66d9247091e6472597eaa8. Signed-off-by: Andrew Purtell --- .../hbase/mapreduce/HFileOutputFormat2.java | 4 +- .../TestHFileOutputFormat2WithSecurity.java | 132 ------------------ .../mapreduce/TestTableMapReduceUtil.java | 46 +++++- .../hadoop/hbase/HBaseTestingUtility.java | 44 ------ 4 files changed, 40 insertions(+), 186 deletions(-) delete mode 100644 hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 2906238edc79..15a3a3ddeb27 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -811,7 +811,7 @@ public static void configureIncrementalLoadMap(Job job, TableDescriptor tableDes * @see #REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY */ - public static void configureRemoteCluster(Job job, Configuration clusterConf) throws IOException { + public static void configureRemoteCluster(Job job, Configuration clusterConf) { Configuration conf = job.getConfiguration(); if (!conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { @@ -828,8 +828,6 @@ public static void configureRemoteCluster(Job job, Configuration clusterConf) th conf.setInt(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY, clientPort); conf.set(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY, parent); - TableMapReduceUtil.initCredentialsForCluster(job, clusterConf); - LOG.info("ZK configs for remote cluster of bulkload is configured: " + quorum + ":" + clientPort + "/" + parent); } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java deleted file mode 100644 index b4cb6a8355fc..000000000000 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import static org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.Closeable; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Tests for {@link HFileOutputFormat2} with secure mode. - */ -@Category({ VerySlowMapReduceTests.class, LargeTests.class }) -public class TestHFileOutputFormat2WithSecurity { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHFileOutputFormat2WithSecurity.class); - - private static final byte[] FAMILIES = Bytes.toBytes("test_cf"); - - private static final String HTTP_PRINCIPAL = "HTTP/localhost"; - - private HBaseTestingUtility utilA; - - private Configuration confA; - - private HBaseTestingUtility utilB; - - private MiniKdc kdc; - - private List clusters = new ArrayList<>(); - - @Before - public void setupSecurityClusters() throws Exception { - utilA = new HBaseTestingUtility(); - confA = utilA.getConfiguration(); - - utilB = new HBaseTestingUtility(); - - // Prepare security configs. - File keytab = new File(utilA.getDataTestDir("keytab").toUri().getPath()); - kdc = utilA.setupMiniKdc(keytab); - String username = UserGroupInformation.getLoginUser().getShortUserName(); - String userPrincipal = username + "/localhost"; - kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); - loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - - // Start security clusterA - clusters.add(utilA.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); - - // Start security clusterB - clusters.add(utilB.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); - } - - @After - public void teardownSecurityClusters() { - IOUtils.closeQuietly(clusters); - clusters.clear(); - if (kdc != null) { - kdc.stop(); - } - } - - @Test - public void testIncrementalLoadInMultiClusterWithSecurity() throws Exception { - TableName tableName = TableName.valueOf("testIncrementalLoadInMultiClusterWithSecurity"); - - // Create table in clusterB - try (Table table = utilB.createTable(tableName, FAMILIES); - RegionLocator r = utilB.getConnection().getRegionLocator(tableName)) { - - // Create job in clusterA - Job job = Job.getInstance(confA, "testIncrementalLoadInMultiClusterWithSecurity"); - job.setWorkingDirectory( - utilA.getDataTestDirOnTestFS("testIncrementalLoadInMultiClusterWithSecurity")); - job.setInputFormatClass(NMapInputFormat.class); - job.setMapperClass(TestHFileOutputFormat2.RandomKVGeneratingMapper.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(KeyValue.class); - HFileOutputFormat2.configureIncrementalLoad(job, table, r); - - assertEquals(2, job.getCredentials().getAllTokens().size()); - - String remoteClusterId = utilB.getHBaseClusterInterface().getClusterMetrics().getClusterId(); - assertTrue(job.getCredentials().getToken(new Text(remoteClusterId)) != null); - } finally { - if (utilB.getAdmin().tableExists(tableName)) { - utilB.deleteTable(tableName); - } - } - } -} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index f661025ac062..3b7392b3ae45 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -29,8 +29,15 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -41,6 +48,7 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.junit.After; @@ -126,6 +134,33 @@ public void testInitTableMapperJob4() throws Exception { assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } + private static Closeable startSecureMiniCluster(HBaseTestingUtility util, MiniKdc kdc, + String principal) throws Exception { + Configuration conf = util.getConfiguration(); + + SecureTestUtil.enableSecurity(conf); + VisibilityTestUtil.enableVisiblityLabels(conf); + SecureTestUtil.verifyConfiguration(conf); + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + AccessController.class.getName() + ',' + TokenProvider.class.getName()); + + HBaseKerberosUtils.setSecuredConfiguration(conf, principal + '@' + kdc.getRealm(), + HTTP_PRINCIPAL + '@' + kdc.getRealm()); + + KerberosName.resetDefaultRealm(); + + util.startMiniCluster(); + try { + util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + } catch (Exception e) { + util.shutdownMiniCluster(); + throw e; + } + + return util::shutdownMiniCluster; + } + @Test public void testInitCredentialsForCluster1() throws Exception { HBaseTestingUtility util1 = new HBaseTestingUtility(); @@ -164,9 +199,8 @@ public void testInitCredentialsForCluster2() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try ( - Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL); - Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { + try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal); + Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); @@ -199,8 +233,7 @@ public void testInitCredentialsForCluster3() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try ( - Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { + try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal)) { try { HBaseTestingUtility util2 = new HBaseTestingUtility(); // Assume util2 is insecure cluster @@ -236,8 +269,7 @@ public void testInitCredentialsForCluster4() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try ( - Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { + try (Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index bc49fe406388..fab033ba7843 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; import edu.umd.cs.findbugs.annotations.Nullable; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -89,7 +88,6 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; @@ -122,12 +120,7 @@ import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.access.AccessController; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; -import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; -import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -156,7 +149,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.ZooKeeper; @@ -399,42 +391,6 @@ public static void closeRegionAndWAL(final HRegion r) throws IOException { r.getWAL().close(); } - /** - * Start mini secure cluster with given kdc and principals. - * @param kdc Mini kdc server - * @param servicePrincipal Service principal without realm. - * @param spnegoPrincipal Spnego principal without realm. - * @return Handler to shutdown the cluster - */ - public Closeable startSecureMiniCluster(MiniKdc kdc, String servicePrincipal, - String spnegoPrincipal) throws Exception { - Configuration conf = getConfiguration(); - - SecureTestUtil.enableSecurity(conf); - VisibilityTestUtil.enableVisiblityLabels(conf); - SecureTestUtil.verifyConfiguration(conf); - - // Reset the static default realm forcibly for hadoop-2.0. - // It has no impact but not required for hadoop-3.0. - KerberosName.resetDefaultRealm(); - - conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - AccessController.class.getName() + ',' + TokenProvider.class.getName()); - - HBaseKerberosUtils.setSecuredConfiguration(conf, servicePrincipal + '@' + kdc.getRealm(), - spnegoPrincipal + '@' + kdc.getRealm()); - - startMiniCluster(); - try { - waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); - } catch (Exception e) { - shutdownMiniCluster(); - throw e; - } - - return this::shutdownMiniCluster; - } - /** * Returns this classes's instance of {@link Configuration}. Be careful how you use the returned * Configuration since {@link Connection} instances can be shared. The Map of Connections is keyed From 9fca7f45cfea26aeaf33708b377a6acdc2f0ae16 Mon Sep 17 00:00:00 2001 From: Dev Hingu Date: Fri, 31 Oct 2025 01:39:14 +0530 Subject: [PATCH 094/336] HBASE-29622 : Flaky Test in TestBackupDelete (#7364) Signed-off-by: Wellington Chevreuil Reviewed-by: Vaibhav Joshi --- .../apache/hadoop/hbase/backup/impl/BackupSystemTable.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index c2253a46d04b..20f83bc2d1ea 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -1247,7 +1247,10 @@ private Get createGetForIncrBackupTableSet(String backupRoot) throws IOException * @return put operation */ private Put createPutForIncrBackupTableSet(Set tables, String backupRoot) { - Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot)); + // added 1ms to prevent LostUpdate problem in case when deleteIncrementalBackupTableSet() + // executed very fast + long ts = EnvironmentEdgeManager.currentTime() + 1; + Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot), ts); for (TableName table : tables) { put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()), EMPTY_VALUE); From eeb44f9dc53088808d6b101fc2d4a0caf3a29f40 Mon Sep 17 00:00:00 2001 From: gvprathyusha6 <70918688+gvprathyusha6@users.noreply.github.com> Date: Tue, 4 Nov 2025 04:38:45 +0530 Subject: [PATCH 095/336] HBASE-29662 - Avoid regionDir/tableDir creation as part of .regioninfo file creation in HRegion initialize (#7406) Signed-off-by: Andrew Purtell Signed-off-by: Viraj Jasani Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java --- .../hadoop/hbase/util/CommonFSUtils.java | 34 +++++- .../TestTableSnapshotInputFormat.java | 101 ++++++++++++++++++ .../hbase/master/janitor/MetaFixer.java | 26 +++++ .../procedure/TruncateRegionProcedure.java | 15 +++ .../hbase/regionserver/HRegionFileSystem.java | 13 ++- .../org/apache/hadoop/hbase/util/FSUtils.java | 28 ++++- .../hadoop/hbase/HBaseTestingUtility.java | 20 ++++ .../TestCoreRegionCoprocessor.java | 2 + .../hbase/master/janitor/TestMetaFixer.java | 1 + .../TestCompactionArchiveConcurrentClose.java | 6 +- .../TestCompactionArchiveIOException.java | 1 + .../hbase/regionserver/TestHRegion.java | 86 +++++++++++++-- .../TestStoreFileRefresherChore.java | 3 +- .../wal/AbstractTestWALReplay.java | 3 +- 14 files changed, 323 insertions(+), 16 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index 73bb6f38cd27..26ed7c982581 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -188,11 +188,39 @@ public static int getDefaultBufferSize(final FileSystem fs) { */ public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm, boolean overwrite) throws IOException { + return create(fs, path, perm, overwrite, true); + } + + /** + * Create the specified file on the filesystem. By default, this will: + *

    + *
  1. apply the umask in the configuration (if it is enabled)
    2. + *
  2. use the fs configured buffer size (or 4096 if not set)
    3. + *
  3. use the default replication
    4. + *
  4. use the default block size
    5. + *
  5. not track progress
    6. + *
+ * @param fs {@link FileSystem} on which to write the file + * @param path {@link Path} to the file to write + * @param perm intial permissions + * @param overwrite Whether or not the created file should be overwritten. + * @param isRecursiveCreate recursively create parent directories + * @return output stream to the created file + * @throws IOException if the file cannot be created + */ + public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm, + boolean overwrite, boolean isRecursiveCreate) throws IOException { if (LOG.isTraceEnabled()) { - LOG.trace("Creating file={} with permission={}, overwrite={}", path, perm, overwrite); + LOG.trace("Creating file={} with permission={}, overwrite={}, recursive={}", path, perm, + overwrite, isRecursiveCreate); + } + if (isRecursiveCreate) { + return fs.create(path, perm, overwrite, getDefaultBufferSize(fs), + getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); + } else { + return fs.createNonRecursive(path, perm, overwrite, getDefaultBufferSize(fs), + getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); } - return fs.create(path, perm, overwrite, getDefaultBufferSize(fs), - getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null); } /** diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index eca275cf0a90..9909ce15af9c 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.TestTableSnapshotScanner; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat.TableSnapshotRegionSplit; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; @@ -583,4 +584,104 @@ public void testCleanRestoreDir() throws Exception { TableSnapshotInputFormat.cleanRestoreDir(job, snapshotName); Assert.assertFalse(fs.exists(restorePath)); } + + /** + * Test that explicitly restores a snapshot to a temp directory and reads the restored regions via + * ClientSideRegionScanner through a MapReduce job. + *

+ * This test verifies the full workflow: 1. Create and load a table with data 2. Create a snapshot + * and restore the snapshot to a temporary directory 3. Configure a job to read the restored + * regions via ClientSideRegionScanner using TableSnapshotInputFormat and verify that it succeeds + * 4. Delete restored temporary directory 5. Configure a new job and verify that it fails + */ + @Test + public void testReadFromRestoredSnapshotViaMR() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final String snapshotName = tableName + "_snapshot"; + try { + if (UTIL.getAdmin().tableExists(tableName)) { + UTIL.deleteTable(tableName); + } + UTIL.createTable(tableName, FAMILIES, new byte[][] { bbb, yyy }); + + Admin admin = UTIL.getAdmin(); + int regionNum = admin.getRegions(tableName).size(); + LOG.info("Created table with {} regions", regionNum); + + Table table = UTIL.getConnection().getTable(tableName); + UTIL.loadTable(table, FAMILIES); + table.close(); + + Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); + FileSystem fs = rootDir.getFileSystem(UTIL.getConfiguration()); + SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, Arrays.asList(FAMILIES), + null, snapshotName, rootDir, fs, true); + Path tempRestoreDir = UTIL.getDataTestDirOnTestFS("restore_" + snapshotName); + RestoreSnapshotHelper.copySnapshotForScanner(UTIL.getConfiguration(), fs, rootDir, + tempRestoreDir, snapshotName); + Assert.assertTrue("Restore directory should exist", fs.exists(tempRestoreDir)); + + Job job = Job.getInstance(UTIL.getConfiguration()); + job.setJarByClass(TestTableSnapshotInputFormat.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + TestTableSnapshotInputFormat.class); + Scan scan = new Scan().withStartRow(getStartRow()).withStopRow(getEndRow()); + Configuration conf = job.getConfiguration(); + conf.set("hbase.TableSnapshotInputFormat.snapshot.name", snapshotName); + conf.set("hbase.TableSnapshotInputFormat.restore.dir", tempRestoreDir.toString()); + conf.setInt("hbase.mapreduce.splits.per.region", 1); + job.setReducerClass(TestTableSnapshotReducer.class); + job.setNumReduceTasks(1); + job.setOutputFormatClass(NullOutputFormat.class); + TableMapReduceUtil.initTableMapperJob(snapshotName, // table name (snapshot name in this case) + scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, + false, false, TableSnapshotInputFormat.class); + TableMapReduceUtil.resetCacheConfig(conf); + Assert.assertTrue(job.waitForCompletion(true)); + Assert.assertTrue(job.isSuccessful()); + + // Now verify that job fails when restore directory is deleted + Assert.assertTrue(fs.delete(tempRestoreDir, true)); + Assert.assertFalse("Restore directory should not exist after deletion", + fs.exists(tempRestoreDir)); + Job failureJob = Job.getInstance(UTIL.getConfiguration()); + failureJob.setJarByClass(TestTableSnapshotInputFormat.class); + TableMapReduceUtil.addDependencyJarsForClasses(failureJob.getConfiguration(), + TestTableSnapshotInputFormat.class); + Configuration failureConf = failureJob.getConfiguration(); + // Configure job to use the deleted restore directory + failureConf.set("hbase.TableSnapshotInputFormat.snapshot.name", snapshotName); + failureConf.set("hbase.TableSnapshotInputFormat.restore.dir", tempRestoreDir.toString()); + failureConf.setInt("hbase.mapreduce.splits.per.region", 1); + failureJob.setReducerClass(TestTableSnapshotReducer.class); + failureJob.setNumReduceTasks(1); + failureJob.setOutputFormatClass(NullOutputFormat.class); + + TableMapReduceUtil.initTableMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, + ImmutableBytesWritable.class, NullWritable.class, failureJob, false, false, + TableSnapshotInputFormat.class); + TableMapReduceUtil.resetCacheConfig(failureConf); + + Assert.assertFalse("Restore directory should not exist before job execution", + fs.exists(tempRestoreDir)); + failureJob.waitForCompletion(true); + + Assert.assertFalse("Job should fail since the restored snapshot directory is deleted", + failureJob.isSuccessful()); + + } finally { + try { + if (UTIL.getAdmin().tableExists(tableName)) { + UTIL.deleteTable(tableName); + } + } catch (Exception e) { + LOG.warn("Error deleting table", e); + } + try { + UTIL.getAdmin().deleteSnapshot(snapshotName); + } catch (Exception e) { + LOG.warn("Error deleting snapshot", e); + } + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java index 77410c3d91ce..1493e4f69b83 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/MetaFixer.java @@ -29,6 +29,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -38,10 +39,13 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.exceptions.MergeRegionException; +import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; @@ -105,6 +109,7 @@ void fixHoles(CatalogJanitorReport report) { final List newRegionInfos = createRegionInfosForHoles(holes); final List newMetaEntries = createMetaEntries(masterServices, newRegionInfos); + createRegionDirectories(masterServices, newMetaEntries); final TransitRegionStateProcedure[] assignProcedures = masterServices.getAssignmentManager().createRoundRobinAssignProcedures(newMetaEntries); @@ -226,6 +231,27 @@ private static List createMetaEntries(final MasterServices masterSer return createMetaEntriesSuccesses; } + private static void createRegionDirectories(final MasterServices masterServices, + final List regions) { + if (regions.isEmpty()) { + return; + } + final MasterFileSystem mfs = masterServices.getMasterFileSystem(); + final Path rootDir = mfs.getRootDir(); + for (RegionInfo regionInfo : regions) { + if (regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { + try { + Path tableDir = CommonFSUtils.getTableDir(rootDir, regionInfo.getTable()); + HRegionFileSystem.createRegionOnFileSystem(masterServices.getConfiguration(), + mfs.getFileSystem(), tableDir, regionInfo); + } catch (IOException e) { + LOG.warn("Failed to create region directory for {}: {}", + regionInfo.getRegionNameAsString(), e.getMessage(), e); + } + } + } + } + /** * Fix overlaps noted in CJ consistency report. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java index 993aca6dd435..7b17439c9447 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java @@ -109,6 +109,7 @@ assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed() setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_ONLINE); break; case TRUNCATE_REGION_MAKE_ONLINE: + createRegionOnFileSystem(env); addChildProcedure(createAssignProcedures(env)); setNextState(TruncateRegionState.TRUNCATE_REGION_POST_OPERATION); break; @@ -130,6 +131,20 @@ assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed() return Flow.HAS_MORE_STATE; } + private void createRegionOnFileSystem(final MasterProcedureEnv env) throws IOException { + RegionStateNode regionNode = + env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); + regionNode.lock(); + try { + final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName()); + HRegionFileSystem.createRegionOnFileSystem(env.getMasterConfiguration(), mfs.getFileSystem(), + tableDir, getRegion()); + } finally { + regionNode.unlock(); + } + } + private void deleteRegionFromFileSystem(final MasterProcedureEnv env) throws IOException { RegionStateNode regionNode = env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index b80599fd61a3..a0d4f8996606 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -809,7 +809,10 @@ private static void writeRegionInfoFileContent(final Configuration conf, final F // First check to get the permissions FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); // Write the RegionInfo file content - try (FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null)) { + // HBASE-29662: Fail .regioninfo file creation, if the region directory doesn't exist, + // avoiding silent masking of missing region directories during region initialization. + // The region directory should already exist when this method is called. + try (FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null, false)) { out.write(content); } } @@ -893,6 +896,14 @@ private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent, final b CommonFSUtils.delete(fs, tmpPath, true); } + // Check parent (region) directory exists first to maintain HBASE-29662 protection + if (!fs.exists(getRegionDir())) { + throw new IOException("Region directory does not exist: " + getRegionDir()); + } + if (!fs.exists(getTempDir())) { + fs.mkdirs(getTempDir()); + } + // Write HRI to a file in case we need to recover hbase:meta writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index ba2010554660..7437143d2ce3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -213,6 +213,32 @@ public static boolean deleteRegionDir(final Configuration conf, final RegionInfo */ public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path path, FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException { + return create(conf, fs, path, perm, favoredNodes, true); + } + + /** + * Create the specified file on the filesystem. By default, this will: + *

    + *
  1. overwrite the file if it exists
    2. + *
  2. apply the umask in the configuration (if it is enabled)
    3. + *
  3. use the fs configured buffer size (or 4096 if not set)
    4. + *
  4. use the configured column family replication or default replication if + * {@link ColumnFamilyDescriptorBuilder#DEFAULT_DFS_REPLICATION}
    5. + *
  5. use the default block size
    6. + *
  6. not track progress
    7. + *
+ * @param conf configurations + * @param fs {@link FileSystem} on which to write the file + * @param path {@link Path} to the file to write + * @param perm permissions + * @param favoredNodes favored data nodes + * @param isRecursiveCreate recursively create parent directories + * @return output stream to the created file + * @throws IOException if the file cannot be created + */ + public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path path, + FsPermission perm, InetSocketAddress[] favoredNodes, boolean isRecursiveCreate) + throws IOException { if (fs instanceof HFileSystem) { FileSystem backingFs = ((HFileSystem) fs).getBackingFs(); if (backingFs instanceof DistributedFileSystem) { @@ -231,7 +257,7 @@ public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path } } - return CommonFSUtils.create(fs, path, perm, true); + return CommonFSUtils.create(fs, path, perm, true, isRecursiveCreate); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index fab033ba7843..7cb985aee961 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -100,6 +100,7 @@ import org.apache.hadoop.hbase.logging.Log4jUtils; import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; @@ -110,6 +111,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -4277,4 +4279,22 @@ public static void await(final long sleepMillis, final BooleanSupplier condition throw e; } } + + public void createRegionDir(RegionInfo hri) throws IOException { + Path rootDir = getDataTestDir(); + Path tableDir = CommonFSUtils.getTableDir(rootDir, hri.getTable()); + Path regionDir = new Path(tableDir, hri.getEncodedName()); + FileSystem fs = getTestFileSystem(); + if (!fs.exists(regionDir)) { + fs.mkdirs(regionDir); + } + } + + public void createRegionDir(RegionInfo regionInfo, MasterFileSystem masterFileSystem) + throws IOException { + Path tableDir = + CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), regionInfo.getTable()); + HRegionFileSystem.createRegionOnFileSystem(conf, masterFileSystem.getFileSystem(), tableDir, + regionInfo); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoreRegionCoprocessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoreRegionCoprocessor.java index 8c5a2d88601d..fb28b8021478 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoreRegionCoprocessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoreRegionCoprocessor.java @@ -76,12 +76,14 @@ public void before() throws IOException { this.rss = new MockRegionServerServices(HTU.getConfiguration()); ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); + HTU.createRegionDir(ri); this.region = HRegion.openHRegion(ri, td, null, HTU.getConfiguration(), this.rss, null); } @After public void after() throws IOException { this.region.close(); + HTU.cleanupTestDir(); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestMetaFixer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestMetaFixer.java index ea4c93d6de20..cd2d948dd06f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestMetaFixer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestMetaFixer.java @@ -170,6 +170,7 @@ private static RegionInfo makeOverlap(MasterServices services, RegionInfo a, Reg throws IOException { RegionInfo overlapRegion = RegionInfoBuilder.newBuilder(a.getTable()) .setStartKey(a.getStartKey()).setEndKey(b.getEndKey()).build(); + TEST_UTIL.createRegionDir(overlapRegion, services.getMasterFileSystem()); MetaTableAccessor.putsToMetaTable(services.getConnection(), Collections.singletonList(MetaTableAccessor.makePutFromRegionInfo(overlapRegion, EnvironmentEdgeManager.currentTime()))); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java index e64ba3eebf10..9f87bfa707a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java @@ -178,9 +178,11 @@ private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOExcep CommonFSUtils.setRootDir(walConf, tableDir); final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName()); HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null); - + Path regionDir = new Path(tableDir, info.getEncodedName()); + if (!fs.getFileSystem().exists(regionDir)) { + fs.getFileSystem().mkdirs(regionDir); + } region.initialize(); - return region; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java index 706b98aeef7c..72a3e2a97aef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java @@ -198,6 +198,7 @@ private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOExcep .rename(eq(new Path(storeDir, ERROR_FILE)), any()); HRegionFileSystem fs = new HRegionFileSystem(conf, errFS, tableDir, info); + fs.createRegionOnFileSystem(conf, fs.getFileSystem(), tableDir, info); final Configuration walConf = new Configuration(conf); CommonFSUtils.setRootDir(walConf, tableDir); final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index d9a735144480..1d53a7d652bd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -7501,12 +7501,13 @@ public void testBulkLoadReplicationEnabled() throws IOException { final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); - htd.addFamily(new HColumnDescriptor(fam1)); - HRegionInfo hri = - new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); - region = - HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); + TableDescriptor tableDescriptor = + TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); + RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); + TEST_UTIL.createRegionDir(hri); + region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), + TEST_UTIL.getConfiguration(), rss, null); assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); @@ -7864,4 +7865,77 @@ public void testRegionOnCoprocessorsWithoutChange() throws IOException { public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { // a empty region coprocessor class } + + /** + * Test for HBASE-29662: HRegion.initialize() should fail when trying to recreate .regioninfo file + * after the region directory has been deleted. This validates that .regioninfo file creation does + * not create parent directories recursively. + */ + @Test + public void testHRegionInitializeFailsWithDeletedRegionDir() throws Exception { + LOG.info("Testing HRegion initialize failure with deleted region directory"); + + TEST_UTIL = new HBaseTestingUtility(); + Configuration conf = TEST_UTIL.getConfiguration(); + Path testDir = TEST_UTIL.getDataTestDir("testHRegionInitFailure"); + FileSystem fs = testDir.getFileSystem(conf); + + // Create table descriptor + TableName tableName = TableName.valueOf("TestHRegionInitWithDeletedDir"); + byte[] family = Bytes.toBytes("info"); + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); + + // Create region info + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).build(); + + Path tableDir = CommonFSUtils.getTableDir(testDir, tableName); + + // Create WAL for the region + WAL wal = HBaseTestingUtility.createWal(conf, testDir, regionInfo); + + try { + // Create region normally (this should succeed and create region directory) + LOG.info("Creating region normally - should succeed"); + HRegion region = HRegion.createHRegion(regionInfo, testDir, conf, htd, wal, true); + + // Verify region directory exists + Path regionDir = new Path(tableDir, regionInfo.getEncodedName()); + assertTrue("Region directory should exist after creation", fs.exists(regionDir)); + + Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); + assertTrue("Region info file should exist after creation", fs.exists(regionInfoFile)); + + // Delete the region directory (simulating external deletion or corruption) + assertTrue(fs.delete(regionDir, true)); + assertFalse("Region directory should not exist after deletion", fs.exists(regionDir)); + + // Try to open/initialize the region again - this should fail + LOG.info("Attempting to re-initialize region with deleted directory - should fail"); + + // Create a new region instance (simulating region server restart or reopen) + HRegion newRegion = HRegion.newHRegion(tableDir, wal, fs, conf, regionInfo, htd, null); + // Try to initialize - this should fail because the regionDir doesn't exist + IOException regionInitializeException = null; + try { + newRegion.initialize(null); + } catch (IOException e) { + regionInitializeException = e; + } + + // Verify the exception is related to missing parent directory + assertNotNull("Exception should be thrown", regionInitializeException); + String exceptionMessage = regionInitializeException.getMessage().toLowerCase(); + assertTrue(exceptionMessage.contains("region directory does not exist")); + assertFalse("Region directory should still not exist after failed initialization", + fs.exists(regionDir)); + + } finally { + if (wal != null) { + wal.close(); + } + TEST_UTIL.cleanupTestDir(); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 5f36d201a753..2ccd5d70ad7b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -122,9 +122,8 @@ private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null); - + fs.createRegionOnFileSystem(walConf, fs.getFileSystem(), tableDir, info); region.initialize(); - return region; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 2ef7ece92db8..d93970c62050 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -818,7 +818,8 @@ public void testSequentialEditLogSeqNum() throws IOException { // Mock the WAL MockWAL wal = createMockWAL(); - + TEST_UTIL.createRegionDir(hri, + TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem()); HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); From 67f414cfed130bf6325570463ed36ea7d45aebc5 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 3 Nov 2025 19:09:09 -0800 Subject: [PATCH 096/336] Preparing hbase release 2.6.4RC1; tagging and updates to CHANGES.md and RELEASENOTES.md Signed-off-by: Andrew Purtell --- CHANGES.md | 7 +++++-- pom.xml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 47dbc41ee716..6196ef7f06cb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ --> # HBASE Changelog -## Release 2.6.4 - Unreleased (as of 2025-10-22) +## Release 2.6.4 - 2025-11-10 @@ -34,6 +34,7 @@ | JIRA | Summary | Priority | Component | |:---- |:---- | :--- |:---- | +| [HBASE-29679](https://issues.apache.org/jira/browse/HBASE-29679) | Suppress stack trace in RpcThrottlingException | Minor | Quotas | | [HBASE-29663](https://issues.apache.org/jira/browse/HBASE-29663) | TimeBasedLimiters should support dynamic configuration refresh | Major | . | | [HBASE-29653](https://issues.apache.org/jira/browse/HBASE-29653) | Build fails on riscv64 due to os-maven-plugin not recognizing RISC-V architecture | Major | build | | [HBASE-29650](https://issues.apache.org/jira/browse/HBASE-29650) | Upgrade tomcat-jasper to 9.0.110 | Major | UI | @@ -55,7 +56,6 @@ | [HBASE-29479](https://issues.apache.org/jira/browse/HBASE-29479) | QuotaCache is not correctly populated until runs of QuotaRefresherChore | Minor | . | | [HBASE-29556](https://issues.apache.org/jira/browse/HBASE-29556) | Display HBCK and CatalogJanitor report errors properly on HBCK Report page | Major | UI | | [HBASE-29431](https://issues.apache.org/jira/browse/HBASE-29431) | Update the 'ExcludeDNs' information with the cause in RS UI | Major | UI | -| [HBASE-29473](https://issues.apache.org/jira/browse/HBASE-29473) | Obtain target cluster's token for cross clusters job | Major | . | | [HBASE-29528](https://issues.apache.org/jira/browse/HBASE-29528) | Support for cellVisibility in Thrift interface | Minor | Thrift | | [HBASE-29290](https://issues.apache.org/jira/browse/HBASE-29290) | Include port number of Region Server in the Replication Status message | Minor | shell | | [HBASE-29469](https://issues.apache.org/jira/browse/HBASE-29469) | Add RPC throttling metrics to RegionServer for quota monitoring | Minor | metrics | @@ -72,6 +72,9 @@ | JIRA | Summary | Priority | Component | |:---- |:---- | :--- |:---- | +| [HBASE-29662](https://issues.apache.org/jira/browse/HBASE-29662) | Reading data via TableSnapshotInputFormat should fail instead of reading no data if restore directory got deleted | Critical | snapshots | +| [HBASE-29622](https://issues.apache.org/jira/browse/HBASE-29622) | Flaky Test : TestBackupDelete#testBackupDeleteUpdatesIncrementalBackupSet | Major | backup&restore | +| [HBASE-29677](https://issues.apache.org/jira/browse/HBASE-29677) | Thread safety in QuotaRefresherChore | Minor | . | | [HBASE-29604](https://issues.apache.org/jira/browse/HBASE-29604) | BackupHFileCleaner uses flawed time based check | Critical | backup&restore | | [HBASE-29629](https://issues.apache.org/jira/browse/HBASE-29629) | Record the quota user name value on metrics for RpcThrottlingExceptions | Minor | Quotas | | [HBASE-29623](https://issues.apache.org/jira/browse/HBASE-29623) | Blocks for CFs with BlockCache disabled may still get cached on write or compaction | Major | BlockCache | diff --git a/pom.xml b/pom.xml index 370173515e8d..68409106bb54 100644 --- a/pom.xml +++ b/pom.xml @@ -523,7 +523,7 @@ - 2.6.5-SNAPSHOT + 2.6.4 false From 3a7cd9423fd7b1cc9f55b82d88e78eaa4e8cef6a Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 3 Nov 2025 19:09:32 -0800 Subject: [PATCH 097/336] Preparing development version 2.6.5-SNAPSHOT Signed-off-by: Andrew Purtell --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 68409106bb54..370173515e8d 100644 --- a/pom.xml +++ b/pom.xml @@ -523,7 +523,7 @@ - 2.6.4 + 2.6.5-SNAPSHOT false From e55413419ffee4b88affc957c99aa37a34e41801 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 4 Nov 2025 14:55:12 +0800 Subject: [PATCH 098/336] Reapply "HBASE-29473 Obtain target cluster's token for cross clusters job (#7198)" This reverts commit 3bed95feb7d218fbca506f192b3271ab7f663aed. --- .../hbase/mapreduce/HFileOutputFormat2.java | 4 +- .../TestHFileOutputFormat2WithSecurity.java | 132 ++++++++++++++++++ .../mapreduce/TestTableMapReduceUtil.java | 46 +----- .../hadoop/hbase/HBaseTestingUtility.java | 44 ++++++ 4 files changed, 186 insertions(+), 40 deletions(-) create mode 100644 hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 15a3a3ddeb27..2906238edc79 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -811,7 +811,7 @@ public static void configureIncrementalLoadMap(Job job, TableDescriptor tableDes * @see #REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY */ - public static void configureRemoteCluster(Job job, Configuration clusterConf) { + public static void configureRemoteCluster(Job job, Configuration clusterConf) throws IOException { Configuration conf = job.getConfiguration(); if (!conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { @@ -828,6 +828,8 @@ public static void configureRemoteCluster(Job job, Configuration clusterConf) { conf.setInt(REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY, clientPort); conf.set(REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY, parent); + TableMapReduceUtil.initCredentialsForCluster(job, clusterConf); + LOG.info("ZK configs for remote cluster of bulkload is configured: " + quorum + ":" + clientPort + "/" + parent); } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java new file mode 100644 index 000000000000..b4cb6a8355fc --- /dev/null +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2WithSecurity.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import static org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.Closeable; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for {@link HFileOutputFormat2} with secure mode. + */ +@Category({ VerySlowMapReduceTests.class, LargeTests.class }) +public class TestHFileOutputFormat2WithSecurity { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHFileOutputFormat2WithSecurity.class); + + private static final byte[] FAMILIES = Bytes.toBytes("test_cf"); + + private static final String HTTP_PRINCIPAL = "HTTP/localhost"; + + private HBaseTestingUtility utilA; + + private Configuration confA; + + private HBaseTestingUtility utilB; + + private MiniKdc kdc; + + private List clusters = new ArrayList<>(); + + @Before + public void setupSecurityClusters() throws Exception { + utilA = new HBaseTestingUtility(); + confA = utilA.getConfiguration(); + + utilB = new HBaseTestingUtility(); + + // Prepare security configs. + File keytab = new File(utilA.getDataTestDir("keytab").toUri().getPath()); + kdc = utilA.setupMiniKdc(keytab); + String username = UserGroupInformation.getLoginUser().getShortUserName(); + String userPrincipal = username + "/localhost"; + kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); + loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); + + // Start security clusterA + clusters.add(utilA.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + + // Start security clusterB + clusters.add(utilB.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)); + } + + @After + public void teardownSecurityClusters() { + IOUtils.closeQuietly(clusters); + clusters.clear(); + if (kdc != null) { + kdc.stop(); + } + } + + @Test + public void testIncrementalLoadInMultiClusterWithSecurity() throws Exception { + TableName tableName = TableName.valueOf("testIncrementalLoadInMultiClusterWithSecurity"); + + // Create table in clusterB + try (Table table = utilB.createTable(tableName, FAMILIES); + RegionLocator r = utilB.getConnection().getRegionLocator(tableName)) { + + // Create job in clusterA + Job job = Job.getInstance(confA, "testIncrementalLoadInMultiClusterWithSecurity"); + job.setWorkingDirectory( + utilA.getDataTestDirOnTestFS("testIncrementalLoadInMultiClusterWithSecurity")); + job.setInputFormatClass(NMapInputFormat.class); + job.setMapperClass(TestHFileOutputFormat2.RandomKVGeneratingMapper.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + job.setMapOutputValueClass(KeyValue.class); + HFileOutputFormat2.configureIncrementalLoad(job, table, r); + + assertEquals(2, job.getCredentials().getAllTokens().size()); + + String remoteClusterId = utilB.getHBaseClusterInterface().getClusterMetrics().getClusterId(); + assertTrue(job.getCredentials().getToken(new Text(remoteClusterId)) != null); + } finally { + if (utilB.getAdmin().tableExists(tableName)) { + utilB.deleteTable(tableName); + } + } + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java index 3b7392b3ae45..f661025ac062 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceUtil.java @@ -29,15 +29,8 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.security.HBaseKerberosUtils; -import org.apache.hadoop.hbase.security.access.AccessController; -import org.apache.hadoop.hbase.security.access.PermissionStorage; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.security.provider.SaslClientAuthenticationProviders; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; -import org.apache.hadoop.hbase.security.token.TokenProvider; -import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -48,7 +41,6 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.junit.After; @@ -134,33 +126,6 @@ public void testInitTableMapperJob4() throws Exception { assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE)); } - private static Closeable startSecureMiniCluster(HBaseTestingUtility util, MiniKdc kdc, - String principal) throws Exception { - Configuration conf = util.getConfiguration(); - - SecureTestUtil.enableSecurity(conf); - VisibilityTestUtil.enableVisiblityLabels(conf); - SecureTestUtil.verifyConfiguration(conf); - - conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - AccessController.class.getName() + ',' + TokenProvider.class.getName()); - - HBaseKerberosUtils.setSecuredConfiguration(conf, principal + '@' + kdc.getRealm(), - HTTP_PRINCIPAL + '@' + kdc.getRealm()); - - KerberosName.resetDefaultRealm(); - - util.startMiniCluster(); - try { - util.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); - } catch (Exception e) { - util.shutdownMiniCluster(); - throw e; - } - - return util::shutdownMiniCluster; - } - @Test public void testInitCredentialsForCluster1() throws Exception { HBaseTestingUtility util1 = new HBaseTestingUtility(); @@ -199,8 +164,9 @@ public void testInitCredentialsForCluster2() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal); - Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL); + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); @@ -233,7 +199,8 @@ public void testInitCredentialsForCluster3() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util1Closeable = startSecureMiniCluster(util1, kdc, userPrincipal)) { + try ( + Closeable util1Closeable = util1.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { HBaseTestingUtility util2 = new HBaseTestingUtility(); // Assume util2 is insecure cluster @@ -269,7 +236,8 @@ public void testInitCredentialsForCluster4() throws Exception { kdc.createPrincipal(keytab, userPrincipal, HTTP_PRINCIPAL); loginUserFromKeytab(userPrincipal + '@' + kdc.getRealm(), keytab.getAbsolutePath()); - try (Closeable util2Closeable = startSecureMiniCluster(util2, kdc, userPrincipal)) { + try ( + Closeable util2Closeable = util2.startSecureMiniCluster(kdc, userPrincipal, HTTP_PRINCIPAL)) { try { Configuration conf1 = util1.getConfiguration(); Job job = Job.getInstance(conf1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 7cb985aee961..48a7615b2a4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -88,6 +89,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; @@ -122,7 +124,12 @@ import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.security.token.TokenProvider; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; +import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -151,6 +158,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.ZooKeeper; @@ -393,6 +401,42 @@ public static void closeRegionAndWAL(final HRegion r) throws IOException { r.getWAL().close(); } + /** + * Start mini secure cluster with given kdc and principals. + * @param kdc Mini kdc server + * @param servicePrincipal Service principal without realm. + * @param spnegoPrincipal Spnego principal without realm. + * @return Handler to shutdown the cluster + */ + public Closeable startSecureMiniCluster(MiniKdc kdc, String servicePrincipal, + String spnegoPrincipal) throws Exception { + Configuration conf = getConfiguration(); + + SecureTestUtil.enableSecurity(conf); + VisibilityTestUtil.enableVisiblityLabels(conf); + SecureTestUtil.verifyConfiguration(conf); + + // Reset the static default realm forcibly for hadoop-2.0. + // It has no impact but not required for hadoop-3.0. + KerberosName.resetDefaultRealm(); + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + AccessController.class.getName() + ',' + TokenProvider.class.getName()); + + HBaseKerberosUtils.setSecuredConfiguration(conf, servicePrincipal + '@' + kdc.getRealm(), + spnegoPrincipal + '@' + kdc.getRealm()); + + startMiniCluster(); + try { + waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); + } catch (Exception e) { + shutdownMiniCluster(); + throw e; + } + + return this::shutdownMiniCluster; + } + /** * Returns this classes's instance of {@link Configuration}. Be careful how you use the returned * Configuration since {@link Connection} instances can be shared. The Map of Connections is keyed From 87b93cd9d4f2e71cadc229841d2400a60f01aa42 Mon Sep 17 00:00:00 2001 From: mokai Date: Tue, 4 Nov 2025 14:50:42 +0800 Subject: [PATCH 099/336] HBASE-29686 Compatible issue of HFileOutputFormat2#configureRemoteCluster (#7415) Signed-off-by: Duo Zhang Signed-off-by: Junegunn Choi Signed-off-by: Pankaj Kumar Reviewed-by: chaijunjie0101 <1340011734@qq.com> (cherry picked from commit eae219812addbcc7a7b59b43df2cff18793f9f7d) --- .../hbase/mapreduce/HFileOutputFormat2.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 2906238edc79..5651bdf75e41 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; import java.io.IOException; +import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; import java.net.URLDecoder; @@ -622,7 +623,7 @@ private static void writePartitions(Configuration conf, Path partitionsPath, public static void configureIncrementalLoad(Job job, Table table, RegionLocator regionLocator) throws IOException { configureIncrementalLoad(job, table.getDescriptor(), regionLocator); - configureRemoteCluster(job, table.getConfiguration()); + configureForRemoteCluster(job, table.getConfiguration()); } /** @@ -810,8 +811,34 @@ public static void configureIncrementalLoadMap(Job job, TableDescriptor tableDes * @see #REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY * @see #REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY + * @deprecated As of release 2.6.4, this will be removed in HBase 4.0.0 Use + * {@link #configureForRemoteCluster(Job, Configuration)} instead. */ - public static void configureRemoteCluster(Job job, Configuration clusterConf) throws IOException { + @Deprecated + public static void configureRemoteCluster(Job job, Configuration clusterConf) { + try { + configureForRemoteCluster(job, clusterConf); + } catch (IOException e) { + LOG.error("Configure remote cluster error.", e); + throw new UncheckedIOException("Configure remote cluster error.", e); + } + } + + /** + * Configure HBase cluster key for remote cluster to load region location for locality-sensitive + * if it's enabled. It's not necessary to call this method explicitly when the cluster key for + * HBase cluster to be used to load region location is configured in the job configuration. Call + * this method when another HBase cluster key is configured in the job configuration. For example, + * you should call when you load data from HBase cluster A using {@link TableInputFormat} and + * generate hfiles for HBase cluster B. Otherwise, HFileOutputFormat2 fetch location from cluster + * A and locality-sensitive won't working correctly. If authentication is enabled, it obtains the + * token for the specific cluster. + * @param job which has configuration to be updated + * @param clusterConf which contains cluster key of the HBase cluster to be locality-sensitive + * @throws IOException Exception while initializing cluster credentials + */ + public static void configureForRemoteCluster(Job job, Configuration clusterConf) + throws IOException { Configuration conf = job.getConfiguration(); if (!conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { From 31efbc0cfe0956fadf1d62aee3df5ad3f5923711 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Tue, 4 Nov 2025 18:37:22 +0100 Subject: [PATCH 100/336] HBASE-29700 Always close RPC servers in AbstractTestIPC (#7435) (cherry picked from commit 5fb9066d646e6b259936b88c4de58e45432436d1) Signed-off-by: Duo Zhang (cherry picked from commit 4ac5b6dfa82ff8c9b62180d3936109b229d815f8) --- .../java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java index e9d0e8de30b3..b18d111c336e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java @@ -581,6 +581,8 @@ public void testTracingSuccessIpc() throws IOException, ServiceException { everyItem(allOf(hasStatusWithCode(StatusCode.OK), hasTraceId(traceRule.getSpans().iterator().next().getTraceId()), hasDuration(greaterThanOrEqualTo(Duration.ofMillis(100L)))))); + } finally { + rpcServer.stop(); } } @@ -609,6 +611,8 @@ public void testTracingErrorIpc() throws IOException { assertFalse("no spans provided", traceRule.getSpans().isEmpty()); assertThat(traceRule.getSpans(), everyItem(allOf(hasStatusWithCode(StatusCode.ERROR), hasTraceId(traceRule.getSpans().iterator().next().getTraceId())))); + } finally { + rpcServer.stop(); } } @@ -670,6 +674,8 @@ public void testGetConnectionRegistry() throws IOException, ServiceException { GetConnectionRegistryResponse resp = stub.getConnectionRegistry(null, GetConnectionRegistryRequest.getDefaultInstance()); assertEquals(clusterId, resp.getClusterId()); + } finally { + rpcServer.stop(); } } @@ -701,6 +707,8 @@ public void testGetConnectionRegistryError() throws IOException, ServiceExceptio assertEquals(FatalConnectionException.class.getName(), ((RemoteException) pcrc.getFailed()).getClassName()); assertThat(pcrc.getFailed().getMessage(), startsWith("Expected HEADER=")); + } finally { + rpcServer.stop(); } } } From 6ddb8e0aae4add032e9c14903bc27aaf95b7448c Mon Sep 17 00:00:00 2001 From: Huginn <63332600+Huginn-kio@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:57:25 +0800 Subject: [PATCH 101/336] HBASE-29667 Correct block priority to SINGLE on the first write to the bucket cache (#7399) Reviewed by: Kota-SH Signed-off-by: Wellington Chevreuil --- .../hadoop/hbase/io/hfile/bucket/BucketEntry.java | 2 +- .../hadoop/hbase/io/hfile/bucket/TestBucketCache.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java index c93dac8a572b..6ee953717341 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketEntry.java @@ -117,7 +117,7 @@ public class BucketEntry implements HBaseReferenceCounted { this.onDiskSizeWithHeader = onDiskSizeWithHeader; this.accessCounter = accessCounter; this.cachedTime = cachedTime; - this.priority = inMemory ? BlockPriority.MEMORY : BlockPriority.MULTI; + this.priority = inMemory ? BlockPriority.MEMORY : BlockPriority.SINGLE; this.refCnt = RefCnt.create(createRecycler.apply(this)); this.markedAsEvicted = new AtomicBoolean(false); this.allocator = allocator; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java index 84731d03fa93..879a32a50366 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; +import org.apache.hadoop.hbase.io.hfile.BlockPriority; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; import org.apache.hadoop.hbase.io.hfile.CacheTestUtils.HFileBlockPair; @@ -1098,4 +1099,13 @@ private BucketCache testEvictOrphans(long orphanEvictionGracePeriod) throws Exce bucketCache.freeSpace("test"); return bucketCache; } + + @Test + public void testBlockPriority() throws Exception { + HFileBlockPair block = CacheTestUtils.generateHFileBlocks(BLOCK_SIZE, 1)[0]; + cacheAndWaitUntilFlushedToBucket(cache, block.getBlockName(), block.getBlock(), true); + assertEquals(cache.backingMap.get(block.getBlockName()).getPriority(), BlockPriority.SINGLE); + cache.getBlock(block.getBlockName(), true, false, true); + assertEquals(cache.backingMap.get(block.getBlockName()).getPriority(), BlockPriority.MULTI); + } } From 5494a42e4a9698dbfd17228f0f8d43ac164d7b4c Mon Sep 17 00:00:00 2001 From: Liu Xiao <42756849+liuxiaocs7@users.noreply.github.com> Date: Wed, 5 Nov 2025 22:13:20 +0800 Subject: [PATCH 102/336] HBASE-29703 Remove duplicate calls to withNextBlockOnDiskSize (#7440) Signed-off-by: Wellington Chevreuil Signed-off-by: Duo Zhang --- .../java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index c963fc2617fc..796b44a7a233 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -285,7 +285,7 @@ public static HFileBlock getBlockForCaching(CacheConfig cacheConf, HFileBlock bl .withOnDiskSizeWithoutHeader(block.getOnDiskSizeWithoutHeader()) .withUncompressedSizeWithoutHeader(block.getUncompressedSizeWithoutHeader()) .withPrevBlockOffset(block.getPrevBlockOffset()).withByteBuff(buff) - .withFillHeader(FILL_HEADER).withOffset(block.getOffset()).withNextBlockOnDiskSize(-1) + .withFillHeader(FILL_HEADER).withOffset(block.getOffset()) .withOnDiskDataSizeWithHeader(block.getOnDiskDataSizeWithHeader() + numBytes) .withHFileContext(cloneContext(block.getHFileContext())) .withNextBlockOnDiskSize(block.getNextBlockOnDiskSize()) From d6aae14a1aa187337015f6754ad0e156f9912912 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 6 Nov 2025 09:31:07 +0100 Subject: [PATCH 103/336] HBASE-29702 Remove shade plugin from hbase-protocol-shaded (#7439) Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang (cherry picked from commit b911715490d10824ade79cbe990a2b53a1997dbb) --- hbase-protocol-shaded/pom.xml | 47 ----------------------------------- 1 file changed, 47 deletions(-) diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml index e96405ae5c8e..8904ef868755 100644 --- a/hbase-protocol-shaded/pom.xml +++ b/hbase-protocol-shaded/pom.xml @@ -108,53 +108,6 @@ com.google.code.maven-replacer-plugin replacer - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - - - shade - - package - - true - true - - - - com.google.protobuf - org.apache.hadoop.hbase.shaded.com.google.protobuf - - - - - - javax.annotation:javax.annotation-api - - org.apache.hbase.thirdparty:* - com.google.protobuf:protobuf-java - com.google.code.findbugs:* - com.google.j2objc:j2objc-annotations - org.codehaus.mojo:animal-sniffer-annotations - junit:junit - commons-logging:commons-logging - org.slf4j:* - org.apache.logging.log4j:* - org.apache.yetus:audience-annotations - com.github.stephenc.fingbugs:* - com.github.spotbugs:* - - - - - - org.apache.maven.plugins maven-checkstyle-plugin From a4de090a65737f096cb4843da64583a60ba54868 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 6 Nov 2025 19:17:04 +0100 Subject: [PATCH 104/336] HBASE-29704 Replace unsupported forkMode failsafe parameter in hbase-it (#7442) Signed-off-by: Nihal Jain (cherry picked from commit 7fd405718a529d37270ba41c6ce3c46925ac7ca3) --- hbase-it/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 743180e45b43..cb437d5be620 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -300,7 +300,7 @@ maven-failsafe-plugin false - always + false 1800 From 4ea77a086d5775d9bd456718ed04096d8ab35d03 Mon Sep 17 00:00:00 2001 From: Istvan Toth Date: Thu, 6 Nov 2025 19:22:28 +0100 Subject: [PATCH 105/336] HBASE-29701 Update README.txt in hbase-protocol-shaded (#7444) Signed-off-by: Duo Zhang Signed-off-by: Nihal Jain (cherry picked from commit 144cd012890f2ad5276039a798f5ee366a85134d) --- hbase-protocol-shaded/README.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hbase-protocol-shaded/README.txt b/hbase-protocol-shaded/README.txt index b0030fac2157..eb5aaebfcd0d 100644 --- a/hbase-protocol-shaded/README.txt +++ b/hbase-protocol-shaded/README.txt @@ -1,6 +1,9 @@ This module has proto files used by core. These protos overlap with protos that are used by coprocessor endpoints -(CPEP) in the module hbase-protocol. So core versions have -a different name, the generated classes are relocated --- i.e. shaded -- to a new location; they are moved from -org.apache.hadoop.hbase.* to org.apache.hadoop.hbase.shaded. +(CPEP) in the module hbase-protocol. To make sure that the +core versions have a different names, these classes are +using a different package; +they are in +org.apache.hadoop.hbase.shaded.protobuf.generated.* +, while the hbase-protocol classes are in +org.apache.hadoop.hbase.protobuf.generated.*. From 68e9204f1429fce1e03617c95e0e83f42f1cbecd Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Wed, 12 Nov 2025 16:15:43 +0530 Subject: [PATCH 106/336] HBASE-29033 Add a shell command for inspecting the state of enable/disable_rpc_throttle (#7448) (#7458) Signed-off-by: Nihal Jain Signed-off-by: Pankaj Kumar Reviewed-by: Vaibhav Joshi (cherry picked from commit 59bd6b2bef6071748db055828d9799bd122e4042) Co-authored-by: Liu Xiao <42756849+liuxiaocs7@users.noreply.github.com> --- hbase-shell/src/main/ruby/hbase/quotas.rb | 4 ++ hbase-shell/src/main/ruby/shell.rb | 1 + .../shell/commands/rpc_throttle_enabled.rb | 42 +++++++++++++++++++ .../src/test/ruby/hbase/quotas_test.rb | 15 +++++++ 4 files changed, 62 insertions(+) create mode 100644 hbase-shell/src/main/ruby/shell/commands/rpc_throttle_enabled.rb diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 3488f0f33dbc..1749e2d956b9 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -375,6 +375,10 @@ def switch_rpc_throttle(enabled) @admin.switchRpcThrottle(java.lang.Boolean.valueOf(enabled)) end + def rpc_throttle_enabled? + @admin.isRpcThrottleEnabled + end + def switch_exceed_throttle_quota(enabled) @admin.exceedThrottleQuotaSwitch(java.lang.Boolean.valueOf(enabled)) end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 37bebbdd492d..70aa7b8ae619 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -574,6 +574,7 @@ def self.exception_handler(hide_traceback) list_snapshot_sizes enable_rpc_throttle disable_rpc_throttle + rpc_throttle_enabled enable_exceed_throttle_quota disable_exceed_throttle_quota ] diff --git a/hbase-shell/src/main/ruby/shell/commands/rpc_throttle_enabled.rb b/hbase-shell/src/main/ruby/shell/commands/rpc_throttle_enabled.rb new file mode 100644 index 000000000000..0e789f7eadea --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/rpc_throttle_enabled.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +module Shell + module Commands + # Prints whether rpc throttle is enabled + class RpcThrottleEnabled < Command + def help + <<-EOF +Query the current rpc throttle state. +Return true if rpc throttle is enabled, false otherwise. + +Examples: + hbase> rpc_throttle_enabled +EOF + end + + def command + state = quotas_admin.rpc_throttle_enabled? + formatter.row([state.to_s]) + state + end + end + end +end diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb index 6e506c52f14a..508749f544f8 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -245,15 +245,30 @@ def teardown end define_test 'switch rpc throttle' do + result = nil + output = capture_stdout { result = command(:rpc_throttle_enabled) } + assert(output.include?('true')) + assert(result == true) + result = nil output = capture_stdout { result = command(:disable_rpc_throttle) } assert(output.include?('Previous rpc throttle state : true')) assert(result == true) + result = nil + output = capture_stdout { result = command(:rpc_throttle_enabled) } + assert(output.include?('false')) + assert(result == false) + result = nil output = capture_stdout { result = command(:enable_rpc_throttle) } assert(output.include?('Previous rpc throttle state : false')) assert(result == false) + + result = nil + output = capture_stdout { result = command(:rpc_throttle_enabled) } + assert(output.include?('true')) + assert(result == true) end define_test 'can set and remove region server quota' do From 4bd754dca5326865e851dace47fbbf2d7a82888b Mon Sep 17 00:00:00 2001 From: Deep Golani <54791570+deepgolani4@users.noreply.github.com> Date: Thu, 13 Nov 2025 09:17:21 -0500 Subject: [PATCH 107/336] HBASE-29568 - Allow for a configurable grace period when using Time Based Priority (#7425) (#7462) Signed-off-by: Wellington Chevreuil --- .../regionserver/DataTieringManager.java | 30 +++++++-- .../regionserver/TestDataTieringManager.java | 62 +++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 2a5e2a5aa39d..6638fd2049c6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -47,6 +47,9 @@ public class DataTieringManager { "hbase.regionserver.datatiering.enable"; public static final boolean DEFAULT_GLOBAL_DATA_TIERING_ENABLED = false; // disabled by default public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type"; + public static final String HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY = + "hbase.hstore.datatiering.grace.period.millis"; + public static final long DEFAULT_DATATIERING_GRACE_PERIOD = 0; public static final String DATATIERING_HOT_DATA_AGE_KEY = "hbase.hstore.datatiering.hot.age.millis"; public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE; @@ -139,6 +142,9 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException { * @return {@code true} if the data is hot, {@code false} otherwise */ public boolean isHotData(long maxTimestamp, Configuration conf) { + if (isWithinGracePeriod(maxTimestamp, conf)) { + return true; + } DataTieringType dataTieringType = getDataTieringType(conf); if ( @@ -170,8 +176,11 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { throw new DataTieringException( "Store file corresponding to " + hFilePath + " doesn't exist"); } - return hotDataValidator(dataTieringType.getInstance().getTimestamp(getHStoreFile(hFilePath)), - getDataTieringHotDataAge(configuration)); + long maxTimestamp = dataTieringType.getInstance().getTimestamp(hStoreFile); + if (isWithinGracePeriod(maxTimestamp, configuration)) { + return true; + } + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; @@ -189,13 +198,21 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) { DataTieringType dataTieringType = getDataTieringType(configuration); if (hFileInfo != null && !dataTieringType.equals(DataTieringType.NONE)) { - return hotDataValidator(dataTieringType.getInstance().getTimestamp(hFileInfo), - getDataTieringHotDataAge(configuration)); + long maxTimestamp = dataTieringType.getInstance().getTimestamp(hFileInfo); + if (isWithinGracePeriod(maxTimestamp, configuration)) { + return true; + } + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; } + private boolean isWithinGracePeriod(long maxTimestamp, Configuration conf) { + long gracePeriod = getDataTieringGracePeriod(conf); + return gracePeriod > 0 && (getCurrentTimestamp() - maxTimestamp) < gracePeriod; + } + private boolean hotDataValidator(long maxTimestamp, long hotDataAge) { long currentTimestamp = getCurrentTimestamp(); long diff = currentTimestamp - maxTimestamp; @@ -275,6 +292,11 @@ private long getDataTieringHotDataAge(Configuration conf) { conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE))); } + private long getDataTieringGracePeriod(Configuration conf) { + return Long.parseLong(conf.get(HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + String.valueOf(DEFAULT_DATATIERING_GRACE_PERIOD))); + } + /* * This API traverses through the list of online regions and returns a subset of these files-names * that are cold. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 507f14a86946..21e2315ae881 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -231,6 +231,57 @@ public void testHotDataWithPath() throws IOException { new DataTieringException("Store file corresponding to " + hFilePath + " doesn't exist")); } + @Test + public void testGracePeriodMakesColdFileHot() throws IOException, DataTieringException { + initializeTestEnvironment(); + + long hotAge = 1 * DAY; + long gracePeriod = 3 * DAY; + + long currentTime = System.currentTimeMillis(); + long fileTimestamp = currentTime - (2 * DAY); + + Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod); + HRegion region = createHRegion("tableGracePeriod", conf); + HStore hStore = createHStore(region, "cf1", conf); + + HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem()); + file.initReader(); + + hStore.refreshStoreFiles(); + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + Path hFilePath = file.getPath(); + assertTrue("File should be hot due to grace period", dataTieringManager.isHotData(hFilePath)); + } + + @Test + public void testFileIsColdWithoutGracePeriod() throws IOException, DataTieringException { + initializeTestEnvironment(); + + long hotAge = 1 * DAY; + long gracePeriod = 0; + long currentTime = System.currentTimeMillis(); + long fileTimestamp = currentTime - (2 * DAY); + + Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod); + HRegion region = createHRegion("tableNoGracePeriod", conf); + HStore hStore = createHStore(region, "cf1", conf); + + HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem()); + file.initReader(); + + hStore.refreshStoreFiles(); + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + + Path hFilePath = file.getPath(); + assertFalse("File should be cold without grace period", + dataTieringManager.isHotData(hFilePath)); + } + @Test public void testPrefetchWhenDataTieringEnabled() throws IOException { setPrefetchBlocksOnOpen(); @@ -770,6 +821,8 @@ private static HRegion createHRegion(String table, Configuration conf) throws IO .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY)) .build(); RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); @@ -797,6 +850,8 @@ private static HStore createHStore(HRegion region, String columnFamily, Configur .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY)) .build(); return new HStore(region, columnFamilyDescriptor, conf, false); @@ -809,6 +864,13 @@ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotData return conf; } + private static Configuration getConfWithGracePeriod(long hotDataAge, long gracePeriod) { + Configuration conf = getConfWithTimeRangeDataTieringEnabled(hotDataAge); + conf.set(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + String.valueOf(gracePeriod)); + return conf; + } + static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp, HRegionFileSystem regionFs) throws IOException { String columnFamily = storeDir.getName(); From e299b85799b19375cf5f291ce565ef784439d57d Mon Sep 17 00:00:00 2001 From: Umesh <9414umeshkumar@gmail.com> Date: Mon, 17 Nov 2025 11:51:30 +0530 Subject: [PATCH 108/336] HBASE-29714 Increase DEFAULT_RS_REMOTE_PROC_RETRY_LIMIT to 10 (#7466) Signed-off-by: Andrew Purtell Signed-off-by: Viraj Jasani Signed-off-by: Aman Poonia --- .../hadoop/hbase/master/procedure/RSProcedureDispatcher.java | 4 ++-- .../java/org/apache/hadoop/hbase/util/TestProcDispatcher.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java index 8bb4df135c61..5ab8a830fd08 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java @@ -261,9 +261,9 @@ protected class ExecuteProceduresRemoteCall implements RemoteProcedureResolver, /** * The default retry limit. Waiting for more than {@value} attempts is not going to help much * for genuine connectivity errors. Therefore, consider fail-fast after {@value} retries. Value - * = {@value} + * = {@value}. 10 retries means we will wait for at least 28.5 seconds before killing RS. */ - private static final int DEFAULT_RS_REMOTE_PROC_RETRY_LIMIT = 5; + private static final int DEFAULT_RS_REMOTE_PROC_RETRY_LIMIT = 10; private final int failFastRetryLimit; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestProcDispatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestProcDispatcher.java index 740a65f2b614..5e42ee2f4a45 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestProcDispatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestProcDispatcher.java @@ -77,6 +77,7 @@ public class TestProcDispatcher { public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set(HBASE_MASTER_RSPROC_DISPATCHER_CLASS, RSProcDispatcher.class.getName()); + TEST_UTIL.getConfiguration().setInt("hbase.master.rs.remote.proc.fail.fast.limit", 5); TEST_UTIL.startMiniCluster(3); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); rs0 = cluster.getRegionServer(0).getServerName(); From 32bd1ab4fdf2ce64b3d5f933d300e211a0e63710 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Mon, 17 Nov 2025 17:31:20 +0000 Subject: [PATCH 109/336] HBASE-29707 Fix region cache % metrics miss calculation (#7467) (cherry picked from commit 290ae1e9f5da06ba883b8f4d9069b646a533c81e) Signed-off-by: Peter Somogyi Reviewed-by: Kevin Geiszler Change-Id: Icc6fd32e3b9e56702a48bfa7ba53cb0a84764e0e --- .../hadoop/hbase/io/hfile/BlockCache.java | 8 ++ .../hadoop/hbase/io/hfile/BlockCacheKey.java | 6 + .../hbase/io/hfile/CombinedBlockCache.java | 5 + .../hbase/io/hfile/HFilePreadReader.java | 2 +- .../hbase/io/hfile/bucket/BucketCache.java | 61 ++++---- .../hadoop/hbase/util/HFileArchiveUtil.java | 12 ++ .../hadoop/hbase/TestSplitWithCache.java | 8 +- .../io/hfile/TestPrefetchWithBucketCache.java | 131 +++++++++++++++++- 8 files changed, 203 insertions(+), 30 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 313b4034fb86..d3b7eb2057ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -85,6 +85,14 @@ Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, */ int evictBlocksByHfileName(String hfileName); + /** + * Evicts all blocks for the given HFile by path. + * @return the number of blocks evicted + */ + default int evictBlocksByHfilePath(Path hfilePath) { + return evictBlocksByHfileName(hfilePath.getName()); + } + /** * Get the statistics for this block cache. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java index bcc1f58ba5e2..f87b456c29bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java @@ -32,6 +32,7 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { private final long offset; private BlockType blockType; private final boolean isPrimaryReplicaBlock; + private Path filePath; /** @@ -116,4 +117,9 @@ public void setBlockType(BlockType blockType) { public Path getFilePath() { return filePath; } + + public void setFilePath(Path filePath) { + this.filePath = filePath; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 856b5da6d117..45301abe08cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -149,6 +149,11 @@ public int evictBlocksByHfileName(String hfileName) { return l1Cache.evictBlocksByHfileName(hfileName) + l2Cache.evictBlocksByHfileName(hfileName); } + @Override + public int evictBlocksByHfilePath(Path hfilePath) { + return l1Cache.evictBlocksByHfilePath(hfilePath) + l2Cache.evictBlocksByHfilePath(hfilePath); + } + @Override public CacheStats getStats() { return this.combinedCacheStats; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java index 3ef5f50db029..86dcdf97065c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePreadReader.java @@ -168,7 +168,7 @@ public void close(boolean evictOnClose) throws IOException { // Deallocate data blocks cacheConf.getBlockCache().ifPresent(cache -> { if (evictOnClose) { - int numEvicted = cache.evictBlocksByHfileName(name); + int numEvicted = cache.evictBlocksByHfilePath(path); if (LOG.isTraceEnabled()) { LOG.trace("On close, file= {} evicted= {} block(s)", name, numEvicted); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 8b333bce0b5a..9174be34b087 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.IdReadWriteLock; import org.apache.hadoop.hbase.util.IdReadWriteLock.ReferenceType; import org.apache.hadoop.hbase.util.Pair; @@ -670,7 +671,7 @@ public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, // the cache map state might differ from the actual cache. If we reach this block, // we should remove the cache key entry from the backing map backingMap.remove(key); - fileNotFullyCached(key.getHfileName()); + fileNotFullyCached(key, bucketEntry); LOG.debug("Failed to fetch block for cache key: {}.", key, hioex); } catch (IOException ioex) { LOG.error("Failed reading block " + key + " from bucket cache", ioex); @@ -695,7 +696,7 @@ void blockEvicted(BlockCacheKey cacheKey, BucketEntry bucketEntry, boolean decre if (decrementBlockNumber) { this.blockNumber.decrement(); if (ioEngine.isPersistent()) { - fileNotFullyCached(cacheKey.getHfileName()); + fileNotFullyCached(cacheKey, bucketEntry); } } if (evictedByEvictionProcess) { @@ -706,23 +707,11 @@ void blockEvicted(BlockCacheKey cacheKey, BucketEntry bucketEntry, boolean decre } } - private void fileNotFullyCached(String hfileName) { - // Update the regionPrefetchedSizeMap before removing the file from prefetchCompleted - if (fullyCachedFiles.containsKey(hfileName)) { - Pair regionEntry = fullyCachedFiles.get(hfileName); - String regionEncodedName = regionEntry.getFirst(); - long filePrefetchSize = regionEntry.getSecond(); - LOG.debug("Removing file {} for region {}", hfileName, regionEncodedName); - regionCachedSize.computeIfPresent(regionEncodedName, (rn, pf) -> pf - filePrefetchSize); - // If all the blocks for a region are evicted from the cache, remove the entry for that region - if ( - regionCachedSize.containsKey(regionEncodedName) - && regionCachedSize.get(regionEncodedName) == 0 - ) { - regionCachedSize.remove(regionEncodedName); - } - } - fullyCachedFiles.remove(hfileName); + private void fileNotFullyCached(BlockCacheKey key, BucketEntry entry) { + // Update the updateRegionCachedSize before removing the file from fullyCachedFiles. + // This computation should happen even if the file is not in fullyCachedFiles map. + updateRegionCachedSize(key.getFilePath(), (entry.getLength() * -1)); + fullyCachedFiles.remove(key.getHfileName()); } public void fileCacheCompleted(Path filePath, long size) { @@ -736,9 +725,19 @@ public void fileCacheCompleted(Path filePath, long size) { private void updateRegionCachedSize(Path filePath, long cachedSize) { if (filePath != null) { - String regionName = filePath.getParent().getParent().getName(); - regionCachedSize.merge(regionName, cachedSize, - (previousSize, newBlockSize) -> previousSize + newBlockSize); + if (HFileArchiveUtil.isHFileArchived(filePath)) { + LOG.trace("Skipping region cached size update for archived file: {}", filePath); + } else { + String regionName = filePath.getParent().getParent().getName(); + regionCachedSize.merge(regionName, cachedSize, + (previousSize, newBlockSize) -> previousSize + newBlockSize); + LOG.trace("Updating region cached size for region: {}", regionName); + // If all the blocks for a region are evicted from the cache, + // remove the entry for that region from regionCachedSize map. + if (regionCachedSize.get(regionName) <= 0) { + regionCachedSize.remove(regionName); + } + } } } @@ -1608,7 +1607,7 @@ private void verifyFileIntegrity(BucketCacheProtos.BucketCacheEntry proto) { } catch (IOException e1) { LOG.debug("Check for key {} failed. Evicting.", keyEntry.getKey()); evictBlock(keyEntry.getKey()); - fileNotFullyCached(keyEntry.getKey().getHfileName()); + fileNotFullyCached(keyEntry.getKey(), keyEntry.getValue()); } } backingMapValidated.set(true); @@ -1854,13 +1853,20 @@ public int evictBlocksByHfileName(String hfileName) { } @Override - public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long endOffset) { - fileNotFullyCached(hfileName); + public int evictBlocksByHfilePath(Path hfilePath) { + return evictBlocksRangeByHfileName(hfilePath.getName(), hfilePath, 0, Long.MAX_VALUE); + } + + public int evictBlocksRangeByHfileName(String hfileName, Path filePath, long initOffset, + long endOffset) { Set keySet = getAllCacheKeysForFile(hfileName, initOffset, endOffset); LOG.debug("found {} blocks for file {}, starting offset: {}, end offset: {}", keySet.size(), hfileName, initOffset, endOffset); int numEvicted = 0; for (BlockCacheKey key : keySet) { + if (filePath != null) { + key.setFilePath(filePath); + } if (evictBlock(key)) { ++numEvicted; } @@ -1868,6 +1874,11 @@ public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long e return numEvicted; } + @Override + public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long endOffset) { + return evictBlocksRangeByHfileName(hfileName, null, initOffset, endOffset); + } + private Set getAllCacheKeysForFile(String hfileName, long init, long end) { return blocksByHFile.subSet(new BlockCacheKey(hfileName, init), true, new BlockCacheKey(hfileName, end), true); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java index 9f26eda12c1f..0731fe654dc4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java @@ -200,4 +200,16 @@ public static TableName getTableName(Path archivePath) { if (p == null) return null; return TableName.valueOf(p.getName(), tbl); } + + public static boolean isHFileArchived(Path path) { + Path currentDir = path; + for (int i = 0; i < 6; i++) { + currentDir = currentDir.getParent(); + if (currentDir == null) { + return false; + } + } + return HConstants.HFILE_ARCHIVE_DIRECTORY.equals(currentDir.getName()); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java index 91e65610f81c..74bd2f5a3796 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSplitWithCache.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -63,10 +64,15 @@ public static void setUp() throws Exception { UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); UTIL.getConfiguration().setBoolean(CACHE_BLOCKS_ON_WRITE_KEY, true); UTIL.getConfiguration().setBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, true); - UTIL.getConfiguration().set(BUCKET_CACHE_IOENGINE_KEY, "offheap"); UTIL.getConfiguration().setInt(BUCKET_CACHE_SIZE_KEY, 200); } + @Before + public void testSetup() { + UTIL.getConfiguration().set(BUCKET_CACHE_IOENGINE_KEY, + "file:" + UTIL.getDataTestDir() + "/bucketcache"); + } + @Test public void testEvictOnSplit() throws Exception { doTest("testEvictOnSplit", true, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index 15fc42656ad4..714c857963a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - import java.io.File; import java.io.IOException; import java.util.Map; @@ -48,6 +47,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.ByteBuffAllocator; +import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.io.hfile.bucket.BucketEntry; import org.apache.hadoop.hbase.regionserver.BloomType; @@ -55,12 +55,14 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -70,7 +72,6 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; @Category({ IOTests.class, MediumTests.class }) @@ -268,7 +269,7 @@ public void testPrefetchMetricProgress() throws Exception { BucketCache bc = BucketCache.getBucketCacheFromCacheConfig(cacheConf).get(); MutableLong regionCachedSize = new MutableLong(0); // Our file should have 6 DATA blocks. We should wait for all of them to be cached - long waitedTime = Waiter.waitFor(conf, 300, () -> { + Waiter.waitFor(conf, 300, () -> { if (bc.getBackingMap().size() > 0) { long currentSize = bc.getRegionCachedInfo().get().get(regionName); assertTrue(regionCachedSize.getValue() <= currentSize); @@ -279,6 +280,130 @@ public void testPrefetchMetricProgress() throws Exception { }); } + @Test + public void testPrefetchMetricProgressForLinks() throws Exception { + conf.setLong(BUCKET_CACHE_SIZE_KEY, 200); + blockCache = BlockCacheFactory.createBlockCache(conf); + cacheConf = new CacheConfig(conf, blockCache); + final RegionInfo hri = + RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ + Configuration testConf = new Configuration(this.conf); + Path testDir = TEST_UTIL.getDataTestDir(name.getMethodName()); + CommonFSUtils.setRootDir(testConf, testDir); + Path tableDir = CommonFSUtils.getTableDir(testDir, hri.getTable()); + RegionInfo region = RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build(); + Path regionDir = new Path(tableDir, region.getEncodedName()); + Path cfDir = new Path(regionDir, "cf"); + HRegionFileSystem regionFS = + HRegionFileSystem.createRegionOnFileSystem(testConf, fs, tableDir, region); + Path storeFile = writeStoreFile(100, cfDir); + // Prefetches the file blocks + LOG.debug("First read should prefetch the blocks."); + readStoreFile(storeFile); + BucketCache bc = BucketCache.getBucketCacheFromCacheConfig(cacheConf).get(); + // Our file should have 6 DATA blocks. We should wait for all of them to be cached + Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6); + long cachedSize = bc.getRegionCachedInfo().get().get(region.getEncodedName()); + + final RegionInfo dstHri = + RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + HRegionFileSystem dstRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, dstHri.getTable()), dstHri); + + Path dstPath = new Path(regionFS.getTableDir(), new Path(dstHri.getRegionNameAsString(), "cf")); + + Path linkFilePath = + new Path(dstPath, HFileLink.createHFileLinkName(region, storeFile.getName())); + + StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(dstPath) + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("cf")) + .withRegionFileSystem(dstRegionFs).build()); + StoreFileInfo sfi = sft.getStoreFileInfo(linkFilePath, true); + + HStoreFile hsf = new HStoreFile(sfi, BloomType.NONE, cacheConf); + assertTrue(sfi.isLink()); + hsf.initReader(); + HFile.Reader reader = hsf.getReader().getHFileReader(); + while (!reader.prefetchComplete()) { + // Sleep for a bit + Thread.sleep(1000); + } + // HFileLink use the path of the target file to create a reader, so it should resolve to the + // already cached blocks and not insert new blocks in the cache. + Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6); + + assertEquals(cachedSize, (long) bc.getRegionCachedInfo().get().get(region.getEncodedName())); + } + + @Test + public void testPrefetchMetricProgressForLinksToArchived() throws Exception { + conf.setLong(BUCKET_CACHE_SIZE_KEY, 200); + blockCache = BlockCacheFactory.createBlockCache(conf); + cacheConf = new CacheConfig(conf, blockCache); + + // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ + Configuration testConf = new Configuration(this.conf); + Path testDir = TEST_UTIL.getDataTestDir(name.getMethodName()); + CommonFSUtils.setRootDir(testConf, testDir); + + final RegionInfo hri = + RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); + Path tableDir = CommonFSUtils.getTableDir(testDir, hri.getTable()); + RegionInfo region = RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build(); + Path regionDir = new Path(tableDir, region.getEncodedName()); + Path cfDir = new Path(regionDir, "cf"); + + Path storeFile = writeStoreFile(100, cfDir); + // Prefetches the file blocks + LOG.debug("First read should prefetch the blocks."); + readStoreFile(storeFile); + BucketCache bc = BucketCache.getBucketCacheFromCacheConfig(cacheConf).get(); + // Our file should have 6 DATA blocks. We should wait for all of them to be cached + Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 6); + long cachedSize = bc.getRegionCachedInfo().get().get(region.getEncodedName()); + + // create another file, but in the archive dir, hence it won't be cached + Path archiveRoot = new Path(testDir, "archive"); + Path archiveTableDir = CommonFSUtils.getTableDir(archiveRoot, hri.getTable()); + Path archiveRegionDir = new Path(archiveTableDir, region.getEncodedName()); + Path archiveCfDir = new Path(archiveRegionDir, "cf"); + Path archivedFile = writeStoreFile(100, archiveCfDir); + + final RegionInfo testRegion = + RegionInfoBuilder.newBuilder(TableName.valueOf(tableDir.getName())).build(); + final HRegionFileSystem testRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, + CommonFSUtils.getTableDir(testDir, testRegion.getTable()), testRegion); + // Just create a link to the archived file + Path dstPath = new Path(tableDir, new Path(testRegion.getEncodedName(), "cf")); + + Path linkFilePath = + new Path(dstPath, HFileLink.createHFileLinkName(region, archivedFile.getName())); + + StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false, + StoreContext.getBuilder().withFamilyStoreDirectoryPath(dstPath) + .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("cf")) + .withRegionFileSystem(testRegionFs).build()); + StoreFileInfo sfi = sft.getStoreFileInfo(linkFilePath, true); + + HStoreFile hsf = new HStoreFile(sfi, BloomType.NONE, cacheConf); + assertTrue(sfi.isLink()); + hsf.initReader(); + HFile.Reader reader = hsf.getReader().getHFileReader(); + while (!reader.prefetchComplete()) { + // Sleep for a bit + Thread.sleep(1000); + } + // HFileLink use the path of the target file to create a reader, but the target file is in the + // archive, so it wasn't cached previously and should be cached when we open the link. + Waiter.waitFor(testConf, 300, () -> bc.getBackingMap().size() == 12); + // cached size for the region of target file shouldn't change + assertEquals(cachedSize, (long) bc.getRegionCachedInfo().get().get(region.getEncodedName())); + // cached size for the region with link pointing to archive dir shouldn't be updated + assertNull(bc.getRegionCachedInfo().get().get(testRegion.getEncodedName())); + } + private void readStoreFile(Path storeFilePath) throws Exception { readStoreFile(storeFilePath, (r, o) -> { HFileBlock block = null; From 8683cf6cc1a93283f7e1c99910eb9f37af203edb Mon Sep 17 00:00:00 2001 From: Kevin Geiszler Date: Fri, 21 Nov 2025 10:40:43 -0800 Subject: [PATCH 110/336] HBASE-29723: Backport "HBASE-25282: Remove processingServers in DeadServer as we can get this information by Procedure of master" to branch-2.6 (#7471) Signed-off-by: Tak Lon (Stephen) Wu --- .../hadoop/hbase/master/DeadServer.java | 48 ------------------- .../hbase/master/MasterRpcServices.java | 15 ++++-- .../hadoop/hbase/master/ServerManager.java | 5 +- .../procedure/ServerCrashProcedure.java | 2 - .../hadoop/hbase/TestRegionRebalancing.java | 2 +- .../io/hfile/TestPrefetchWithBucketCache.java | 2 + .../hadoop/hbase/master/TestDeadServer.java | 20 ++------ .../hbase/master/procedure/TestHBCKSCP.java | 1 - 8 files changed, 20 insertions(+), 75 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java index 84d660e66ee6..9467512fc66c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java @@ -33,8 +33,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; - /** * Class to hold dead servers list and utility querying dead server list. Servers are added when * they expire or when we find them in filesystem on startup. When a server crash procedure is @@ -56,12 +54,6 @@ public class DeadServer { */ private final Map deadServers = new HashMap<>(); - /** - * Set of dead servers currently being processed by a SCP. Added to this list at the start of SCP - * and removed after it is done processing the crash. - */ - private final Set processingServers = new HashSet<>(); - /** * @param serverName server name. * @return true if this server is on the dead servers list false otherwise @@ -70,15 +62,6 @@ public synchronized boolean isDeadServer(final ServerName serverName) { return deadServers.containsKey(serverName); } - /** - * Checks if there are currently any dead servers being processed by the master. Returns true if - * at least one region server is currently being processed as dead. - * @return true if any RS are being processed as dead - */ - synchronized boolean areDeadServersInProgress() { - return !processingServers.isEmpty(); - } - public synchronized Set copyServerNames() { Set clone = new HashSet<>(deadServers.size()); clone.addAll(deadServers.keySet()); @@ -90,29 +73,6 @@ public synchronized Set copyServerNames() { */ synchronized void putIfAbsent(ServerName sn) { this.deadServers.putIfAbsent(sn, EnvironmentEdgeManager.currentTime()); - processing(sn); - } - - /** - * Add sn< to set of processing deadservers. - * @see #finish(ServerName) - */ - public synchronized void processing(ServerName sn) { - if (processingServers.add(sn)) { - // Only log on add. - LOG.debug("Processing {}; numProcessing={}", sn, processingServers.size()); - } - } - - /** - * Complete processing for this dead server. - * @param sn ServerName for the dead server. - * @see #processing(ServerName) - */ - public synchronized void finish(ServerName sn) { - if (processingServers.remove(sn)) { - LOG.debug("Removed {} from processing; numProcessing={}", sn, processingServers.size()); - } } public synchronized int size() { @@ -171,17 +131,12 @@ public synchronized String toString() { // Display unified set of servers from both maps Set servers = new HashSet<>(); servers.addAll(deadServers.keySet()); - servers.addAll(processingServers); StringBuilder sb = new StringBuilder(); for (ServerName sn : servers) { if (sb.length() > 0) { sb.append(", "); } sb.append(sn.toString()); - // Star entries that are being processed - if (processingServers.contains(sn)) { - sb.append("*"); - } } return sb.toString(); } @@ -220,9 +175,6 @@ public synchronized Date getTimeOfDeath(final ServerName deadServerName) { * @return true if this server was removed */ public synchronized boolean removeDeadServer(final ServerName deadServerName) { - Preconditions.checkState(!processingServers.contains(deadServerName), - "Asked to remove server still in processingServers set " + deadServerName + " (numProcessing=" - + processingServers.size() + ")"); return this.deadServers.remove(deadServerName) != null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 194cdcca65b2..5b3213d70491 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -2527,11 +2527,18 @@ public ClearDeadServersResponse clearDeadServers(RpcController controller, LOG.debug("Some dead server is still under processing, won't clear the dead server list"); response.addAllServerName(request.getServerNameList()); } else { + DeadServer deadServer = master.getServerManager().getDeadServers(); for (HBaseProtos.ServerName pbServer : request.getServerNameList()) { - if ( - !master.getServerManager().getDeadServers() - .removeDeadServer(ProtobufUtil.toServerName(pbServer)) - ) { + ServerName server = ProtobufUtil.toServerName(pbServer); + final boolean deadInProcess = + master.getProcedures().stream().anyMatch(p -> (p instanceof ServerCrashProcedure) + && ((ServerCrashProcedure) p).getServerName().equals(server)); + if (deadInProcess) { + throw new ServiceException( + String.format("Dead server '%s' is not 'dead' in fact...", server)); + } + + if (!deadServer.removeDeadServer(server)) { response.addServerName(pbServer); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index f7115a5cefb1..075fb78198fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; +import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -547,8 +548,8 @@ public DeadServer getDeadServers() { * Checks if any dead servers are currently in progress. * @return true if any RS are being processed as dead, false if not */ - public boolean areDeadServersInProgress() { - return this.deadservers.areDeadServersInProgress(); + public boolean areDeadServersInProgress() throws IOException { + return master.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure); } void letRegionServersShutdown() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java index 69ebd0567de4..2a7e96149df6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java @@ -140,7 +140,6 @@ protected Flow executeFromState(MasterProcedureEnv env, ServerCrashState state) // This adds server to the DeadServer processing list but not to the DeadServers list. // Server gets removed from processing list below on procedure successful finish. if (!notifiedDeadServer) { - services.getServerManager().getDeadServers().processing(serverName); notifiedDeadServer = true; } @@ -255,7 +254,6 @@ protected Flow executeFromState(MasterProcedureEnv env, ServerCrashState state) case SERVER_CRASH_FINISH: LOG.info("removed crashed server {} after splitting done", serverName); services.getAssignmentManager().getRegionStates().removeServer(serverName); - services.getServerManager().getDeadServers().finish(serverName); updateProgress(true); return Flow.NO_MORE_STATE; default: diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index 1cc0dd8379b5..d9fbe57785be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -183,7 +183,7 @@ public void testRebalanceOnRegionServerNumberChange() throws IOException, Interr /** * Wait on crash processing. Balancer won't run if processing a crashed server. */ - private void waitOnCrashProcessing() { + private void waitOnCrashProcessing() throws IOException { while (UTIL.getHBaseCluster().getMaster().getServerManager().areDeadServersInProgress()) { LOG.info("Waiting on processing of crashed server before proceeding..."); Threads.sleep(1000); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java index 714c857963a1..688802c28e25 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetchWithBucketCache.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.util.Map; @@ -72,6 +73,7 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; @Category({ IOTests.class, MediumTests.class }) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java index 046c72050baa..94ac1823ac36 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.List; import java.util.Set; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -69,22 +70,10 @@ public static void tearDownAfterClass() throws Exception { public void testIsDead() { DeadServer ds = new DeadServer(); ds.putIfAbsent(hostname123); - ds.processing(hostname123); - assertTrue(ds.areDeadServersInProgress()); - ds.finish(hostname123); - assertFalse(ds.areDeadServersInProgress()); ds.putIfAbsent(hostname1234); - ds.processing(hostname1234); - assertTrue(ds.areDeadServersInProgress()); - ds.finish(hostname1234); - assertFalse(ds.areDeadServersInProgress()); ds.putIfAbsent(hostname12345); - ds.processing(hostname12345); - assertTrue(ds.areDeadServersInProgress()); - ds.finish(hostname12345); - assertFalse(ds.areDeadServersInProgress()); // Already dead = 127.0.0.1,9090,112321 // Coming back alive = 127.0.0.1,9090,223341 @@ -104,7 +93,7 @@ public void testIsDead() { } @Test - public void testCrashProcedureReplay() { + public void testCrashProcedureReplay() throws IOException { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); final ProcedureExecutor pExecutor = master.getMasterProcedureExecutor(); ServerCrashProcedure proc = @@ -112,7 +101,7 @@ public void testCrashProcedureReplay() { ProcedureTestingUtility.submitAndWait(pExecutor, proc); - assertFalse(master.getServerManager().getDeadServers().areDeadServersInProgress()); + assertTrue(master.getServerManager().areDeadServersInProgress()); } @Test @@ -163,17 +152,14 @@ public void testClearDeadServer() { d.putIfAbsent(hostname1234); Assert.assertEquals(2, d.size()); - d.finish(hostname123); d.removeDeadServer(hostname123); Assert.assertEquals(1, d.size()); - d.finish(hostname1234); d.removeDeadServer(hostname1234); Assert.assertTrue(d.isEmpty()); d.putIfAbsent(hostname1234); Assert.assertFalse(d.removeDeadServer(hostname123_2)); Assert.assertEquals(1, d.size()); - d.finish(hostname1234); Assert.assertTrue(d.removeDeadServer(hostname1234)); Assert.assertTrue(d.isEmpty()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java index 2303cb1965e5..367c68213a22 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestHBCKSCP.java @@ -145,7 +145,6 @@ public void test() throws Exception { cluster.killRegionServer(rsServerName); master.getServerManager().moveFromOnlineToDeadServers(rsServerName); - master.getServerManager().getDeadServers().finish(rsServerName); master.getServerManager().getDeadServers().removeDeadServer(rsServerName); master.getAssignmentManager().getRegionStates().removeServer(rsServerName); // Kill the server. Nothing should happen since an 'Unknown Server' as far From a4c9922f7d4b80f360dfffdaa257bf83943b1907 Mon Sep 17 00:00:00 2001 From: Chandra Sekhar K Date: Sat, 22 Nov 2025 16:39:31 +0530 Subject: [PATCH 111/336] HBASE-29145 Table Stats shows store file size as zero always for hbase:meta table (#6859) Signed-off-by: Duo Zhang --- .../src/main/resources/hbase-webapps/master/table.jsp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 0a61488cf673..ba05905dcc19 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -380,6 +380,10 @@ double rSize = load.getStoreFileSize().get(Size.Unit.BYTE); if (rSize > 0) { fileSize = StringUtils.byteDesc((long) rSize); + // use the primary replica only for the total store file size calculation + if (j == 0) { + totalStoreFileSizeMB += load.getStoreFileSize().get(Size.Unit.MEGABYTE); + } } double rSizeUncompressed = load.getUncompressedStoreFileSize().get(Size.Unit.BYTE); if (rSizeUncompressed > 0) { From f8d9cd3da3e3f44e402e9e31feffcf7de8895491 Mon Sep 17 00:00:00 2001 From: "Tak Lon (Stephen) Wu" Date: Tue, 25 Nov 2025 02:37:59 +0800 Subject: [PATCH 112/336] HBASE-29724: Backport missing changes of "HBASE-25334: TestRSGroupsFallback.testFallback is flaky" into branch-2 and branch-2.6 (#7472) #7476 (#7476) The change in HBASE-25334 (introduced by commit 32c4432 in branch-2 and branch-2.6) has diverged between the master (#2728) and branch-2 branches. This is a following up changes after HBASE-29720 and have the complete functional changes of HBASE-25334 that uses SCP as the source of dead servers in progress. Signed-off-by: Istvan Toth Co-authored-by: Kevin Geiszler --- .../hadoop/hbase/rsgroup/TestRSGroupsFallback.java | 13 +++++-------- .../apache/hadoop/hbase/master/ServerManager.java | 3 ++- .../apache/hadoop/hbase/master/TestDeadServer.java | 9 ++++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java index dee907fea623..12cc5df1cab1 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java @@ -101,16 +101,13 @@ public void testFallback() throws Exception { assertRegionsInGroup(tableName, FALLBACK_GROUP); // add a new server to default group, regions move to default group - JVMClusterUtil.RegionServerThread t = - TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); - Address startRSAddress = t.getRegionServer().getServerName().getAddress(); - TEST_UTIL.waitFor(3000, - () -> rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).containsServer(startRSAddress)); + TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); assertTrue(master.balance().isBalancerRan()); assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP); // add a new server to test group, regions move back - t = TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); + JVMClusterUtil.RegionServerThread t = + TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); rsGroupAdmin.moveServers( Collections.singleton(t.getRegionServer().getServerName().getAddress()), groupName); assertTrue(master.balance().isBalancerRan()); @@ -119,11 +116,11 @@ public void testFallback() throws Exception { TEST_UTIL.deleteTable(tableName); } - private void assertRegionsInGroup(TableName tableName, String group) throws IOException { + private void assertRegionsInGroup(TableName table, String group) throws IOException { ProcedureTestingUtility .waitAllProcedures(TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(group); - master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName).forEach(region -> { + master.getAssignmentManager().getRegionStates().getRegionsOfTable(table).forEach(region -> { Address regionOnServer = master.getAssignmentManager().getRegionStates() .getRegionAssignments().get(region).getAddress(); assertTrue(groupInfo.getServers().contains(regionOnServer)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 075fb78198fe..443ef742f4fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -549,7 +549,8 @@ public DeadServer getDeadServers() { * @return true if any RS are being processed as dead, false if not */ public boolean areDeadServersInProgress() throws IOException { - return master.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure); + return master.getProcedures().stream() + .anyMatch(p -> !p.isFinished() && p instanceof ServerCrashProcedure); } void letRegionServersShutdown() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java index 94ac1823ac36..ecad5a193435 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.List; import java.util.Set; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -93,15 +92,19 @@ public void testIsDead() { } @Test - public void testCrashProcedureReplay() throws IOException { + public void testCrashProcedureReplay() throws Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); final ProcedureExecutor pExecutor = master.getMasterProcedureExecutor(); ServerCrashProcedure proc = new ServerCrashProcedure(pExecutor.getEnvironment(), hostname123, false, false); + pExecutor.stop(); ProcedureTestingUtility.submitAndWait(pExecutor, proc); - assertTrue(master.getServerManager().areDeadServersInProgress()); + + ProcedureTestingUtility.restart(pExecutor); + ProcedureTestingUtility.waitProcedure(pExecutor, proc); + assertFalse(master.getServerManager().areDeadServersInProgress()); } @Test From b23f5b57718e29d7744d111ba0cf49d87fad2922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Paksy?= Date: Wed, 26 Nov 2025 10:14:23 +0100 Subject: [PATCH 113/336] HBASE-29223 Migrate Master Status Jamon page back to JSP (#6875) (#7479) * HBASE-29223 Migrate Master Status Jamon page back to JSP (#6875) The JSP code is equivalent to the Jamon code, just changed the syntax back to JSP. Request attributes are used to transfer data between JSP pages. Tried to preserve the code as much as possible but did some changes: Sub-templates were usually extracted to separate JSP file (and included with ` (cherry picked from commit be400115fb8bec896f4ff1bc048b2ff63a05b49d) * [ADDENDUM] HBASE-29223 Fix TestMasterStatusUtil (#7416) TestMasterStatusUtil.testGetFragmentationInfoTurnedOn failed in master nightly build Signed-off-by: Nihal Jain Signed-off-by: Duo Zhang (cherry picked from commit 8ef271f837397688c26e215f2eee6e80408eb799) --- .../master/AssignmentManagerStatusTmpl.jamon | 128 --- .../tmpl/master/BackupMasterStatusTmpl.jamon | 70 -- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 808 ------------------ .../hbase/tmpl/master/RSGroupListTmpl.jamon | 393 --------- .../tmpl/master/RegionServerListTmpl.jamon | 545 ------------ .../tmpl/master/RegionVisualizerTmpl.jamon | 119 --- .../tmpl/regionserver/RSStatusTmpl.jamon | 2 +- .../master/http/MasterStatusConstants.java | 40 + .../master/http/MasterStatusServlet.java | 62 +- .../hbase/master/http/MasterStatusUtil.java | 89 ++ .../hbase/master/http/RegionVisualizer.java | 2 +- .../master/assignmentManagerStatus.jsp | 117 +++ .../master/backupMasterStatus.jsp | 66 ++ .../hbase-webapps/master/catalogTables.jsp | 84 ++ .../master/deadRegionServers.jsp | 96 +++ .../resources/hbase-webapps/master/header.jsp | 4 +- .../resources/hbase-webapps/master/index.html | 2 +- .../resources/hbase-webapps/master/master.jsp | 144 +++- .../hbase-webapps/master/peerConfigs.jsp | 82 ++ .../hbase-webapps/master/regionServerList.jsp | 90 ++ .../master/regionServerListBaseStats.jsp | 134 +++ .../regionServerListCompactionStats.jsp | 78 ++ .../master/regionServerListEmptyStat.jsp | 38 + .../master/regionServerListMemoryStats.jsp | 90 ++ .../regionServerListReplicationStats.jsp | 90 ++ .../master/regionServerListRequestStats.jsp | 75 ++ .../master/regionServerListStoreStats.jsp | 108 +++ .../hbase-webapps/master/regionVisualizer.jsp | 120 +++ .../resources/hbase-webapps/master/rits.jsp | 6 +- .../hbase-webapps/master/rsGroupList.jsp | 79 ++ .../master/rsGroupListBaseStats.jsp | 101 +++ .../master/rsGroupListCompactStats.jsp | 73 ++ .../master/rsGroupListMemoryStats.jsp | 85 ++ .../master/rsGroupListRequestStats.jsp | 66 ++ .../master/rsGroupListStoreStats.jsp | 102 +++ .../master/softwareAttributes.jsp | 173 ++++ .../hbase-webapps/master/tablesDetailed.jsp | 9 +- .../hbase-webapps/master/taskMonitor.jsp | 90 ++ .../master/taskMonitorRenderTasks.jsp | 87 ++ .../hbase-webapps/master/userTables.jsp | 117 +++ .../hbase-webapps/master/warnings.jsp | 86 ++ .../static/js/masterStatusInit.js | 126 +++ .../apache/hadoop/hbase/TestInfoServers.java | 5 +- .../master/http/TestMasterStatusPage.java | 187 ++++ .../master/http/TestMasterStatusServlet.java | 159 ---- .../master/http/TestMasterStatusUtil.java | 256 ++++++ 46 files changed, 3185 insertions(+), 2298 deletions(-) delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon delete mode 100644 hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusConstants.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusUtil.java create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/assignmentManagerStatus.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/backupMasterStatus.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/catalogTables.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/deadRegionServers.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/peerConfigs.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerList.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListBaseStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListCompactionStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListEmptyStat.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListMemoryStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListReplicationStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListRequestStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionServerListStoreStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/regionVisualizer.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupList.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupListBaseStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupListCompactStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupListMemoryStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupListRequestStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/rsGroupListStoreStats.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/softwareAttributes.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/taskMonitor.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/taskMonitorRenderTasks.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/userTables.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/master/warnings.jsp create mode 100644 hbase-server/src/main/resources/hbase-webapps/static/js/masterStatusInit.js create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestMasterStatusPage.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestMasterStatusServlet.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestMasterStatusUtil.java diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon deleted file mode 100644 index ee899a7340dc..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon +++ /dev/null @@ -1,128 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%import> -java.util.Map; -java.util.Set; -java.util.SortedSet; -java.util.concurrent.atomic.AtomicInteger; -java.util.stream.Collectors; -org.apache.hadoop.conf.Configuration; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.HConstants; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.client.RegionInfo; -org.apache.hadoop.hbase.client.RegionInfoDisplay; -org.apache.hadoop.hbase.master.RegionState; -org.apache.hadoop.hbase.master.assignment.AssignmentManager; -org.apache.hadoop.hbase.master.assignment.AssignmentManager.RegionInTransitionStat; -org.apache.hadoop.hbase.master.assignment.RegionStates.RegionFailedOpen; -org.apache.hadoop.hbase.util.Pair; - -<%args> -AssignmentManager assignmentManager; -int limit = 100; - - -<%java> -SortedSet rit = assignmentManager.getRegionStates() - .getRegionsInTransitionOrderedByTimestamp(); - - -<%if !rit.isEmpty() %> -<%java> -long currentTime = System.currentTimeMillis(); -RegionInTransitionStat ritStat = assignmentManager.computeRegionInTransitionStat(); - -int numOfRITs = rit.size(); -int ritsPerPage = Math.min(5, numOfRITs); -int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage); - -
-

Regions in Transition

-

<% numOfRITs %> region(s) in transition. - <%if ritStat.hasRegionsTwiceOverThreshold() %> - - <%elseif ritStat.hasRegionsOverThreshold() %> - - <%else> - - - <% ritStat.getTotalRITsOverThreshold() %> region(s) in transition for - more than <% ritStat.getRITThreshold() %> milliseconds. - -

-
-
- <%java int recordItr = 0; %> - <%for RegionState rs : rit %> - <%if (recordItr % ritsPerPage) == 0 %> - <%if recordItr == 0 %> -
- <%else> -
- - - - - - <%if ritStat.isRegionTwiceOverThreshold(rs.getRegion()) %> - - <%elseif ritStat.isRegionOverThreshold(rs.getRegion()) %> - - <%else> - - - <%java> - String retryStatus = "0"; - RegionFailedOpen regionFailedOpen = assignmentManager - .getRegionStates().getFailedOpen(rs.getRegion()); - if (regionFailedOpen != null) { - retryStatus = Integer.toString(regionFailedOpen.getRetries()); - } else if (rs.getState() == RegionState.State.FAILED_OPEN) { - retryStatus = "Failed"; - } - - - - - - <%java recordItr++; %> - <%if (recordItr % ritsPerPage) == 0 %> -
RegionStateRIT time (ms) Retries
<% rs.getRegion().getEncodedName() %> - <% RegionInfoDisplay.getDescriptiveNameFromRegionStateForDisplay(rs, - assignmentManager.getConfiguration()) %><% (currentTime - rs.getStamp()) %> <% retryStatus %>
-
- - - - <%if (recordItr % ritsPerPage) != 0 %> - <%for ; (recordItr % ritsPerPage) != 0 ; recordItr++ %> - - - -
- -
- - - -
-
- - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon deleted file mode 100644 index 21af264bbe34..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/BackupMasterStatusTmpl.jamon +++ /dev/null @@ -1,70 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -HMaster master; - -<%import> -java.util.*; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.ClusterMetrics; -org.apache.hadoop.hbase.master.HMaster; -org.apache.hbase.thirdparty.com.google.common.base.Preconditions; - -<%if (!master.isActiveMaster()) %> - <%java> - ServerName active_master = master.getActiveMaster().orElse(null); - Preconditions.checkState(active_master != null, "Failed to retrieve active master's ServerName!"); - int activeInfoPort = master.getActiveMasterInfoPort(); - -
-
-

Backup Master <% master.getServerName().getHostname() %>

-
-
-

Current Active Master: <% active_master.getHostname() %>

-<%else> -

Backup Masters

- - - - - - - - <%java> - Collection backup_masters = master.getBackupMasters(); - ServerName [] backupServerNames = backup_masters.toArray(new ServerName[backup_masters.size()]); - Arrays.sort(backupServerNames); - for (ServerName serverName : backupServerNames) { - int infoPort = master.getBackupMasterInfoPort(serverName); - - - - - - - <%java> - } - - -
ServerNamePortStart Time
<% serverName.getHostname() %> - <% serverName.getPort() %><% new Date(serverName.getStartcode()) %>
Total:<% backupServerNames.length %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon deleted file mode 100644 index d1b3f8719d2e..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ /dev/null @@ -1,808 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -<%args> -HMaster master; -Map frags = null; -ServerName metaLocation = null; -List servers = null; -Set deadServers = null; -boolean catalogJanitorEnabled = true; -String filter = "general"; -String format = "html"; -ServerManager serverManager = null; -AssignmentManager assignmentManager = null; - -<%import> -java.util.*; -java.net.URLEncoder; -java.io.IOException; -org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil; -org.apache.hadoop.hbase.client.RegionInfo; -org.apache.hadoop.hbase.client.TableDescriptor; -org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -org.apache.hadoop.hbase.replication.ReplicationPeerDescription; -org.apache.hadoop.hbase.HBaseConfiguration; -org.apache.hadoop.hbase.HConstants; -org.apache.hadoop.hbase.HTableDescriptor; -org.apache.hadoop.hbase.NamespaceDescriptor; -org.apache.hadoop.hbase.ServerName; -org.apache.hadoop.hbase.TableName; -org.apache.hadoop.hbase.RSGroupTableAccessor; -org.apache.hadoop.hbase.client.Admin; -org.apache.hadoop.hbase.client.MasterSwitchType; -org.apache.hadoop.hbase.client.TableState; -org.apache.hadoop.hbase.master.assignment.AssignmentManager; -org.apache.hadoop.hbase.master.DeadServer; -org.apache.hadoop.hbase.master.HMaster; -org.apache.hadoop.hbase.master.RegionState; -org.apache.hadoop.hbase.master.ServerManager; -org.apache.hadoop.hbase.rsgroup.RSGroupInfo; -org.apache.hadoop.hbase.protobuf.ProtobufUtil; -org.apache.hadoop.hbase.quotas.QuotaUtil; -org.apache.hadoop.hbase.security.access.PermissionStorage; -org.apache.hadoop.hbase.security.visibility.VisibilityConstants; -org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -org.apache.hadoop.hbase.tool.CanaryTool; -org.apache.hadoop.hbase.util.Bytes; -org.apache.hadoop.hbase.util.CommonFSUtils; -org.apache.hadoop.hbase.util.JvmVersion; -org.apache.hadoop.hbase.util.PrettyPrinter; -org.apache.hadoop.util.StringUtils; -org.apache.hadoop.hbase.util.Strings; - - -<%if format.equals("json") %> - <& ../common/TaskMonitorTmpl; filter = filter; format = "json" &> - <%java return; %> - -<%java> -ServerManager serverManager = master.getServerManager(); -AssignmentManager assignmentManager = master.getAssignmentManager(); - - -<%class> - public String formatZKString() { - StringBuilder quorums = new StringBuilder(); - String zkQuorum = master.getZooKeeper().getQuorum(); - - if (null == zkQuorum) { - return quorums.toString(); - } - - String[] zks = zkQuorum.split(","); - - if (zks.length == 0) { - return quorums.toString(); - } - - for(int i = 0; i < zks.length; ++i) { - quorums.append(zks[i].trim()); - - if (i != (zks.length - 1)) { - quorums.append("
"); - } - } - - return quorums.toString(); - } - - -<%class> - public static String getUserTables(HMaster master, List tables){ - if (master.isInitialized()){ - try { - Map descriptorMap = master.getTableDescriptors().getAll(); - if (descriptorMap != null) { - for (TableDescriptor desc : descriptorMap.values()) { - if (!desc.getTableName().isSystemTable()) { - tables.add(desc); - } - } - } - } catch (IOException e) { - return "Got user tables error, " + e.getMessage(); - } - } - return null; - } - - - - - - - - <%if master.isActiveMaster() %>Master: <%else>Backup Master: </%if> - <% master.getServerName().getHostname() %> - - - - - - - - - - - -
- <%if master.isActiveMaster() %> -
-
-

Master <% master.getServerName().getHostname() %>

-
-
- -
- - <%if JvmVersion.isBadJvmVersion() %> - - - <%if master.isInitialized() && !catalogJanitorEnabled %> - - - <%if master.isInMaintenanceMode() %> - - - <%if !master.isBalancerOn() %> - - - <%if !master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) %> - - - <%if !master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) %> - - - <%if master.getAssignmentManager() != null %> - <& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&> - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - serverManager.getOnlineServersList().size() > 0 %> -
-

RSGroup

- <& RSGroupListTmpl; master= master; serverManager= serverManager&> -
- - -
-
-
-

Region Servers

- <& RegionServerListTmpl; master= master; servers = servers &> - - <%if (deadServers != null) %> - <& deadRegionServers &> - -
-
-
-
- <& BackupMasterStatusTmpl; master = master &> -
-
-
-
-

Tables

-
- -
-
- <%if (metaLocation != null) %> - <& userTables &> - -
-
- <%if (metaLocation != null) %> - <& catalogTables &> - -
-
-
-
-
-
-
-
-
-

Region Visualizer

- <& RegionVisualizerTmpl &> -
-
-
-
-

Peers

- <& peerConfigs &> -
-
- <%else> -
- <& BackupMasterStatusTmpl; master = master &> -
- - - -
- <& ../common/TaskMonitorTmpl; filter = filter; parent = "/master-status" &> -
- -
-

Software Attributes

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <%escape #n> - - - - - - - - - - - - - - - - - - - - - - - - <%if master.isActiveMaster() %> - - - - - - - - - - - - - - - - <%if frags != null %> - - - - - - - - - - - - - - - - - -
Attribute NameValueDescription
JVM Version<% JvmVersion.getVersion() %>JVM vendor and version
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
HBase Source Checksum<% org.apache.hadoop.hbase.util.VersionInfo.getSrcChecksum() %>HBase source SHA512 checksum
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
Hadoop Source Checksum<% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %>Hadoop source MD5 checksum
ZooKeeper Client Version<% org.apache.zookeeper.Version.getVersion() %>, revision=<% org.apache.zookeeper.Version.getRevisionHash() %>ZooKeeper client version and revision hash
ZooKeeper Client Compiled<% org.apache.zookeeper.Version.getBuildDate() %>When ZooKeeper client version was compiled
ZooKeeper Quorum <% formatZKString() %> Addresses of all registered ZK servers. For more, see zk dump.
ZooKeeper Base Path <% master.getZooKeeper().getZNodePaths().baseZNode %>Root node of this cluster in ZK.
Cluster Key <% formatZKString() %>:<% master.getZooKeeper().getZNodePaths().baseZNode %>Key to add this cluster as a peer for replication. Use 'help "add_peer"' in the shell for details.
HBase Root Directory<% CommonFSUtils.getRootDir(master.getConfiguration()).toString() %>Location of HBase home directory
HMaster Start Time<% new Date(master.getMasterStartTime()) %>Date stamp of when this HMaster was started
HMaster Active Time<% new Date(master.getMasterActiveTime()) %>Date stamp of when this HMaster became active
HBase Cluster ID<% master.getClusterId() != null ? master.getClusterId() : "Not set" %>Unique identifier generated for each HBase cluster
Load average<% master.getServerManager() == null ? "0.00" : - StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>Average number of regions per regionserver. Naive computation.
Fragmentation<% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including hbase:meta
Coprocessors<% master.getMasterCoprocessorHost() == null ? "[]" : - java.util.Arrays.toString(master.getMasterCoprocessors()) %>Coprocessors currently loaded by the master
LoadBalancer<% master.getLoadBalancerClassName() %>LoadBalancer to be used in the Master
-
-
-
- - - - - - - - - - - -<%def catalogTables> -<%java> - List sysTables = master.isInitialized() ? - master.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR) : null; - -<%if (sysTables != null && sysTables.size() > 0)%> - - - - <%if (frags != null) %> - - - - -<%for TableDescriptor systemTable : sysTables%> - -<%java>TableName tableName = systemTable.getTableName(); - - <%if (frags != null)%> - - - <%java>String description = null; - if (tableName.equals(TableName.META_TABLE_NAME)){ - description = "The hbase:meta table holds references to all User Table regions."; - } else if (tableName.equals(CanaryTool.DEFAULT_WRITE_TABLE_NAME)){ - description = "The hbase:canary table is used to sniff the write availbility of" - + " each regionserver."; - } else if (tableName.equals(PermissionStorage.ACL_TABLE_NAME)){ - description = "The hbase:acl table holds information about acl."; - } else if (tableName.equals(VisibilityConstants.LABELS_TABLE_NAME)){ - description = "The hbase:labels table holds information about visibility labels."; - } else if (tableName.equals(TableName.NAMESPACE_TABLE_NAME)){ - description = "The hbase:namespace table holds information about namespaces."; - } else if (tableName.equals(QuotaUtil.QUOTA_TABLE_NAME)){ - description = "The hbase:quota table holds quota information about number" + - " or size of requests in a given time frame."; - } else if (tableName.equals(TableName.valueOf("hbase:rsgroup"))){ - description = "The hbase:rsgroup table holds information about regionserver groups."; - } else if (tableName.equals(TableName.valueOf("hbase:replication"))) { - description = "The hbase:replication table tracks cross cluster replication through " + - "WAL file offsets."; - } - - - - -
Table NameFrag.Description
<% tableName %><% frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()) - .intValue() + "%" : "n/a" %><% description %>
- - - -<%def userTables> -<%java> - List tables = new ArrayList(); - String errorMessage = getUserTables(master, tables); - -<%if (tables.size() == 0 && errorMessage != null)%> -

<% errorMessage %>

- - -<%if (tables != null && tables.size() > 0)%> - - - - - - <%if (frags != null) %> - - - - - - - - - - - - - - - - - - <%for TableDescriptor desc : tables%> - <%java> - HTableDescriptor htDesc = new HTableDescriptor(desc); - TableName tableName = htDesc.getTableName(); - TableState tableState = master.getTableStateManager().getTableState(tableName); - Map> tableRegions = - master.getAssignmentManager().getRegionStates() - .getRegionByStateOfTable(tableName); - int openRegionsCount = tableRegions.get(RegionState.State.OPEN).size(); - int openingRegionsCount = tableRegions.get(RegionState.State.OPENING).size(); - int closedRegionsCount = tableRegions.get(RegionState.State.CLOSED).size(); - int closingRegionsCount = tableRegions.get(RegionState.State.CLOSING).size(); - int offlineRegionsCount = tableRegions.get(RegionState.State.OFFLINE).size(); - int splitRegionsCount = tableRegions.get(RegionState.State.SPLIT).size(); - int otherRegionsCount = 0; - for (List list: tableRegions.values()) { - otherRegionsCount += list.size(); - } - // now subtract known states - otherRegionsCount = otherRegionsCount - openRegionsCount - - offlineRegionsCount - splitRegionsCount - - openingRegionsCount - closedRegionsCount - - closingRegionsCount; - String encodedTableName = URLEncoder.encode(tableName.getNameAsString()); - - - - <%if (tableState.isDisabledOrDisabling()) %> <%else> - <%if (frags != null) %> - - - <%if (tableState.isDisabledOrDisabling()) %> <%else> - - <%if (openingRegionsCount > 0) %> <%else> - <%if (closedRegionsCount > 0) %> <%else> - <%if (closingRegionsCount > 0) %> <%else> - <%if (offlineRegionsCount > 0) %> <%else> - <%if (splitRegionsCount > 0) %> <%else> - - - - -

<% tables.size() %> table(s) in set. [Details]. Click count below to - see list of regions currently in 'state' designated by the column title. For 'Other' Region state, - browse to hbase:meta and adjust filter on 'Meta Entries' to - query on states other than those listed here. Queries may take a while if the hbase:meta table - is large.

- -
NamespaceNameFrag.StateRegionsDescription
OPENOPENINGCLOSEDCLOSINGOFFLINESPLITOther
<% tableName.getNamespaceAsString() %>><% URLEncoder.encode(tableName.getQualifierAsString()) %>><% URLEncoder.encode(tableName.getQualifierAsString()) %> <% frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()).intValue() + "%" : "n/a" %><% tableState.getState().name() %><% tableState.getState() %> <% openRegionsCount %><% openingRegionsCount %><% openingRegionsCount %> <% closedRegionsCount %><% closedRegionsCount %> <% closingRegionsCount %><% closingRegionsCount %> <% offlineRegionsCount %><% offlineRegionsCount %> <% splitRegionsCount %><% splitRegionsCount %> <% otherRegionsCount %><% htDesc.toStringCustomizedValues() %>
- - - - - -<%def deadRegionServers> - -<%if (deadServers != null && deadServers.size() > 0)%> -

Dead Region Servers

- - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0 %> - - - - <%java> - List groups = null; - DeadServer deadServerUtil = master.getServerManager().getDeadServers(); - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - } - ServerName [] deadServerNames = deadServers.toArray(new ServerName[deadServers.size()]); - Arrays.sort(deadServerNames); - for (ServerName deadServerName: deadServerNames) { - String rsGroupName = null; - if (groups != null){ - for (RSGroupInfo rsGroupInfo : groups) { - if (rsGroupInfo.containsServer(deadServerName.getAddress())) { - rsGroupName = rsGroupInfo.getName(); - break; - } - } - if (rsGroupName == null) { - rsGroupName = RSGroupInfo.DEFAULT_GROUP; - } - } - - - - - - <%if rsGroupName != null %> - - - - <%java> - } - - - - - - -
ServerNameStop timeRSGroup
<% deadServerName %><% deadServerUtil.getTimeOfDeath(deadServerName) %><% rsGroupName %>
Total: servers: <% deadServers.size() %>
- - - -<%def peerConfigs> -<%java> - List peers = null; - if (master.getReplicationPeerManager() != null) { - peers = master.getReplicationPeerManager().listPeers(null); - } - - - - - - - - - - - - - - - -<%if (peers != null && peers.size() > 0)%> - <%for ReplicationPeerDescription peer : peers %> - <%java> - String peerId = peer.getPeerId(); - ReplicationPeerConfig peerConfig = peer.getPeerConfig(); - - - - - - - - - - - - - - - - - -
Peer IdCluster KeyEndpointStateIsSerialBandwidthReplicateAllNamespacesExclude NamespacesTable CfsExclude Table Cfs
<% peerId %><% peerConfig.getClusterKey() %><% peerConfig.getReplicationEndpointImpl() %><% peer.isEnabled() ? "ENABLED" : "DISABLED" %><% peerConfig.isSerial() %><% peerConfig.getBandwidth() == 0? "UNLIMITED" : Strings.humanReadableInt(peerConfig.getBandwidth()) %><% peerConfig.replicateAllUserTables() %> - <% peerConfig.getNamespaces() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getNamespaces()).replaceAll(";", "; ") %> - - <% peerConfig.getExcludeNamespaces() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getExcludeNamespaces()).replaceAll(";", "; ") %> - - <% peerConfig.getTableCFsMap() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getTableCFsMap()).replaceAll(";", "; ") %> - - <% peerConfig.getExcludeTableCFsMap() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getExcludeTableCFsMap()).replaceAll(";", "; ") %> -
Total: <% (peers != null) ? peers.size() : 0 %>
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon deleted file mode 100644 index a00576786dbf..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon +++ /dev/null @@ -1,393 +0,0 @@ -<%doc> -Copyright The Apache Software Foundation - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -<%args> -HMaster master; -ServerManager serverManager; - - -<%import> - java.util.Collections; - java.util.List; - java.util.Map; - java.util.Set; - java.util.stream.Collectors; - org.apache.hadoop.hbase.master.HMaster; - org.apache.hadoop.hbase.RegionMetrics; - org.apache.hadoop.hbase.ServerMetrics; - org.apache.hadoop.hbase.Size; - org.apache.hadoop.hbase.RSGroupTableAccessor; - org.apache.hadoop.hbase.master.ServerManager; - org.apache.hadoop.hbase.net.Address; - org.apache.hadoop.hbase.rsgroup.RSGroupInfo; - org.apache.hadoop.util.StringUtils; - org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - -<%java> -List groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - -<%if (groups != null && groups.size() > 0)%> - -<%java> -RSGroupInfo [] rsGroupInfos = groups.toArray(new RSGroupInfo[groups.size()]); -Map collectServers = Collections.emptyMap(); -if (master.getServerManager() != null) { - collectServers = - master.getServerManager().getOnlineServers().entrySet().stream() - .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue)); -} - - -
- -
-
- <& rsgroup_baseStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_memoryStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_requestStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_storeStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
- <& rsgroup_compactStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &> -
-
-
- - - -<%def rsgroup_baseStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - - - -<%java> - int totalOnlineServers = 0; - int totalDeadServers = 0; - int totalTables = 0; - int totalRequests = 0; - int totalRegions = 0; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - int onlineServers = 0; - int deadServers = 0; - int tables = 0; - long requestsPerSecond = 0; - int numRegionsOnline = 0; - Set
servers = rsGroupInfo.getServers(); - for (Address server : servers) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - requestsPerSecond += sl.getRequestCountPerSecond(); - numRegionsOnline += sl.getRegionMetrics().size(); - //rsgroup total - totalRegions += sl.getRegionMetrics().size(); - totalRequests += sl.getRequestCountPerSecond(); - totalOnlineServers++; - onlineServers++; - } else { - totalDeadServers++; - deadServers++; - } - } - tables = rsGroupInfo.getTables().size(); - totalTables += tables; - double avgLoad = onlineServers == 0 ? 0 : - (double)numRegionsOnline / (double)onlineServers; - -
- - - - - - - - -<%java> -} - - - - - - - - - -
RSGroup NameNum. Online ServersNum. Dead ServersNum. TablesRequests Per SecondNum. RegionsAverage Load
<& rsGroupLink; rsGroupName=rsGroupName; &><% onlineServers %><% deadServers %><% tables %><% requestsPerSecond %><% numRegionsOnline %><% StringUtils.limitDecimalTo2(avgLoad) %>
Total:<% rsGroupInfos.length %><% totalOnlineServers %><% totalDeadServers %><% totalTables %><% totalRequests %><% totalRegions %><% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>
- - -<%def rsgroup_memoryStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - -<%java> - final String ZEROMB = "0 MB"; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String usedHeapStr = ZEROMB; - String maxHeapStr = ZEROMB; - String memstoreSizeStr = ZEROMB; - String rsGroupName = rsGroupInfo.getName(); - long usedHeap = 0; - long maxHeap = 0; - long memstoreSize = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - usedHeap += (long) sl.getUsedHeapSize().get(Size.Unit.MEGABYTE); - maxHeap += (long) sl.getMaxHeapSize().get(Size.Unit.MEGABYTE); - memstoreSize += (long) sl.getRegionMetrics().values().stream().mapToDouble( - rm -> rm.getMemStoreSize().get(Size.Unit.MEGABYTE)).sum(); - } - } - - if (usedHeap > 0) { - usedHeapStr = TraditionalBinaryPrefix.long2String(usedHeap - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (maxHeap > 0) { - maxHeapStr = TraditionalBinaryPrefix.long2String(maxHeap - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (memstoreSize > 0) { - memstoreSizeStr = TraditionalBinaryPrefix.long2String(memstoreSize - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - - - - - - - -<%java> -} - -
RSGroup NameUsed HeapMax HeapMemstore Size
<& rsGroupLink; rsGroupName=rsGroupName; &><% usedHeapStr %><% maxHeapStr %><% memstoreSizeStr %>
- - -<%def rsgroup_requestStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - -<%java> - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - long requestsPerSecond = 0; - long readRequests = 0; - long writeRequests = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rm : sl.getRegionMetrics().values()) { - readRequests += rm.getReadRequestCount(); - writeRequests += rm.getWriteRequestCount(); - } - requestsPerSecond += sl.getRequestCountPerSecond(); - } - } - - - - - - - -<%java> -} - -
RSGroup NameRequest Per SecondRead Request CountWrite Request Count
<& rsGroupLink; rsGroupName=rsGroupName; &><% requestsPerSecond %><% readRequests %><% writeRequests %>
- - - -<%def rsgroup_storeStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - - - -<%java> - final String ZEROKB = "0 KB"; - final String ZEROMB = "0 MB"; - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String uncompressedStorefileSizeStr = ZEROMB; - String storefileSizeStr = ZEROMB; - String indexSizeStr = ZEROKB; - String bloomSizeStr = ZEROKB; - String rsGroupName = rsGroupInfo.getName(); - int numStores = 0; - long numStorefiles = 0; - long uncompressedStorefileSize = 0; - long storefileSize = 0; - long indexSize = 0; - long bloomSize = 0; - int count = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rm : sl.getRegionMetrics().values()) { - numStores += rm.getStoreCount(); - numStorefiles += rm.getStoreFileCount(); - uncompressedStorefileSize += rm.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); - storefileSize += rm.getStoreFileSize().get(Size.Unit.MEGABYTE); - indexSize += rm.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); - bloomSize += rm.getBloomFilterSize().get(Size.Unit.KILOBYTE); - } - count++; - } - } - if (uncompressedStorefileSize > 0) { - uncompressedStorefileSizeStr = TraditionalBinaryPrefix. - long2String(uncompressedStorefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (storefileSize > 0) { - storefileSizeStr = TraditionalBinaryPrefix. - long2String(storefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (indexSize > 0) { - indexSizeStr = TraditionalBinaryPrefix. - long2String(indexSize * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - if (bloomSize > 0) { - bloomSizeStr = TraditionalBinaryPrefix. - long2String(bloomSize * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - - - - - - - - - - -<%java> -} - -
RSGroup NameNum. StoresNum. StorefilesStorefile Size UncompressedStorefile SizeIndex SizeBloom Size
<& rsGroupLink; rsGroupName=rsGroupName; &><% numStores %><% numStorefiles %><% uncompressedStorefileSizeStr %><% storefileSizeStr %><% indexSizeStr %><% bloomSizeStr %>
- - -<%def rsgroup_compactStats> -<%args> - RSGroupInfo [] rsGroupInfos; - Map collectServers; - - - - - - - - - -<%java> - for (RSGroupInfo rsGroupInfo: rsGroupInfos) { - String rsGroupName = rsGroupInfo.getName(); - int numStores = 0; - long totalCompactingCells = 0; - long totalCompactedCells = 0; - long remainingCells = 0; - long compactionProgress = 0; - for (Address server : rsGroupInfo.getServers()) { - ServerMetrics sl = collectServers.get(server); - if (sl != null) { - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - totalCompactingCells += rl.getCompactingCellCount(); - totalCompactedCells += rl.getCompactedCellCount(); - } - } - } - remainingCells = totalCompactingCells - totalCompactedCells; - String percentDone = ""; - if (totalCompactingCells > 0) { - percentDone = String.format("%.2f", 100 * - ((float) totalCompactedCells / totalCompactingCells)) + "%"; - } - - - - - - - - -<%java> -} - -
RSGroup NameNum. Compacting CellsNum. Compacted CellsRemaining CellsCompaction Progress
<& rsGroupLink; rsGroupName=rsGroupName; &><% totalCompactingCells %><% totalCompactedCells %><% remainingCells %><% percentDone %>
- - - -<%def rsGroupLink> - <%args> - String rsGroupName; - - ><% rsGroupName %> - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon deleted file mode 100644 index 2f1da65bf690..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +++ /dev/null @@ -1,545 +0,0 @@ -<%doc> -Copyright The Apache Software Foundation - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -<%args> -List servers = null; -HMaster master; - - -<%import> - java.util.*; - org.apache.hadoop.hbase.master.HMaster; - org.apache.hadoop.hbase.procedure2.util.StringUtils; - org.apache.hadoop.hbase.replication.ReplicationLoadSource; - org.apache.hadoop.hbase.RegionMetrics; - org.apache.hadoop.hbase.RSGroupTableAccessor; - org.apache.hadoop.hbase.ServerMetrics; - org.apache.hadoop.hbase.ServerName; - org.apache.hadoop.hbase.Size; - org.apache.hadoop.hbase.net.Address; - org.apache.hadoop.hbase.rsgroup.RSGroupInfo; - org.apache.hadoop.hbase.util.VersionInfo; - org.apache.hadoop.hbase.util.Pair; - org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; - - -<%if (servers != null && servers.size() > 0)%> - -<%java> -ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]); -Arrays.sort(serverNames); - - -
- -
-
- <& baseStats; serverNames = serverNames; &> -
-
- <& memoryStats; serverNames = serverNames; &> -
-
- <& requestStats; serverNames = serverNames; &> -
-
- <& storeStats; serverNames = serverNames; &> -
-
- <& compactionStats; serverNames = serverNames; &> -
-
- <& replicationStats; serverNames = serverNames; &> -
-
-
- - - -<%def baseStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - master.getServerManager().getOnlineServersList().size() > 0 %> - - - - - - -<%java> - int totalRegions = 0; - int totalRequestsPerSecond = 0; - int inconsistentNodeNum = 0; - String state = "Normal"; - String masterVersion = VersionInfo.getVersion(); - Set decommissionedServers = new HashSet<>(master.listDecommissionedRegionServers()); - - String rsGroupName = "default"; - List groups; - Map server2GroupMap = new HashMap<>(); - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); - groups.forEach(group -> { - group.getServers().forEach(address -> server2GroupMap.put(address, group)); - }); - } - - for (ServerName serverName: serverNames) { - if (decommissionedServers.contains(serverName)) { - state = "Decommissioned"; - } - ServerMetrics sl = master.getServerManager().getLoad(serverName); - String version = master.getRegionServerVersion(serverName); - if (!masterVersion.equals(version)) { - inconsistentNodeNum ++; - } - - double requestsPerSecond = 0.0; - int numRegionsOnline = 0; - long lastContact = 0; - - if (sl != null) { - requestsPerSecond = sl.getRequestCountPerSecond(); - numRegionsOnline = sl.getRegionMetrics().size(); - totalRegions += sl.getRegionMetrics().size(); - totalRequestsPerSecond += sl.getRequestCountPerSecond(); - lastContact = (System.currentTimeMillis() - sl.getReportTimestamp())/1000; - } - long startcode = serverName.getStartcode(); - - if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null - && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null - && master.getServerManager().getOnlineServersList().size() > 0) { - rsGroupName = server2GroupMap.get(serverName.getAddress()).getName(); - } - - - - - - - - - - <%if !master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null %> - <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null && - master.getServerManager().getOnlineServersList().size() > 0 %> - - - - -<%java> -} - - - - - - -<%if inconsistentNodeNum > 0%> - -<%else> - - - - - -
ServerNameStateStart timeLast contactVersionRequests Per SecondNum. RegionsRSGroup
<& serverNameLink; serverName=serverName; &><% state %><% new Date(startcode) %><% TraditionalBinaryPrefix.long2String(lastContact, "s", 1) %><% version %><% String.format("%,.0f", requestsPerSecond) %><% String.format("%,d", numRegionsOnline) %><% rsGroupName %>
Total:<% servers.size() %><% inconsistentNodeNum %> nodes with inconsistent version<% totalRequestsPerSecond %><% totalRegions %>
- - -<%def memoryStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -final String ZEROMB = "0 MB"; -for (ServerName serverName: serverNames) { - String usedHeapStr = ZEROMB; - String maxHeapStr = ZEROMB; - String memStoreSizeMBStr = ZEROMB; - ServerMetrics sl = master.getServerManager().getLoad(serverName); - if (sl != null) { - long memStoreSizeMB = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - memStoreSizeMB += rl.getMemStoreSize().get(Size.Unit.MEGABYTE); - } - if (memStoreSizeMB > 0) { - memStoreSizeMBStr = TraditionalBinaryPrefix.long2String(memStoreSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - double usedHeapSizeMB = sl.getUsedHeapSize().get(Size.Unit.MEGABYTE); - if (usedHeapSizeMB > 0) { - usedHeapStr = TraditionalBinaryPrefix.long2String((long) usedHeapSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - double maxHeapSizeMB = sl.getMaxHeapSize().get(Size.Unit.MEGABYTE); - if (maxHeapSizeMB > 0) { - maxHeapStr = TraditionalBinaryPrefix.long2String((long) maxHeapSizeMB - * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameUsed HeapMax HeapMemstore Size
<& serverNameLink; serverName=serverName; &><% usedHeapStr %><% maxHeapStr %><% memStoreSizeMBStr %>
- - - -<%def requestStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -for (ServerName serverName: serverNames) { - -ServerMetrics sl = master.getServerManager().getLoad(serverName); -if (sl != null) { - long readRequestCount = 0; - long writeRequestCount = 0; - long filteredReadRequestCount = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - readRequestCount += rl.getReadRequestCount(); - writeRequestCount += rl.getWriteRequestCount(); - filteredReadRequestCount += rl.getFilteredReadRequestCount(); - } - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameRequest Per SecondRead Request CountFiltered Read Request CountWrite Request Count
<& serverNameLink; serverName=serverName; &><% String.format("%,d", sl.getRequestCountPerSecond()) %><% String.format("%,d", readRequestCount) %><% String.format("%,d", filteredReadRequestCount) %><% String.format("%,d", writeRequestCount) %>
- - - -<%def storeStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - - - -<%java> -final String ZEROKB = "0 KB"; -final String ZEROMB = "0 MB"; -for (ServerName serverName: serverNames) { - - String storeUncompressedSizeMBStr = ZEROMB; - String storeFileSizeMBStr = ZEROMB; - String totalStaticIndexSizeKBStr = ZEROKB; - String totalStaticBloomSizeKBStr = ZEROKB; - ServerMetrics sl = master.getServerManager().getLoad(serverName); - if (sl != null) { - long storeCount = 0; - long storeFileCount = 0; - long storeUncompressedSizeMB = 0; - long storeFileSizeMB = 0; - long totalStaticIndexSizeKB = 0; - long totalStaticBloomSizeKB = 0; - for (RegionMetrics rl : sl.getRegionMetrics().values()) { - storeCount += rl.getStoreCount(); - storeFileCount += rl.getStoreFileCount(); - storeUncompressedSizeMB += rl.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); - storeFileSizeMB += rl.getStoreFileSize().get(Size.Unit.MEGABYTE); - totalStaticIndexSizeKB += rl.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); - totalStaticBloomSizeKB += rl.getBloomFilterSize().get(Size.Unit.KILOBYTE); - } - if (storeUncompressedSizeMB > 0) { - storeUncompressedSizeMBStr = TraditionalBinaryPrefix. - long2String(storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (storeFileSizeMB > 0) { - storeFileSizeMBStr = TraditionalBinaryPrefix. - long2String(storeFileSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); - } - if (totalStaticIndexSizeKB > 0) { - totalStaticIndexSizeKBStr = TraditionalBinaryPrefix. - long2String(totalStaticIndexSizeKB * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - if (totalStaticBloomSizeKB > 0) { - totalStaticBloomSizeKBStr = TraditionalBinaryPrefix. - long2String(totalStaticBloomSizeKB * TraditionalBinaryPrefix.KILO.value, "B", 1); - } - - - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameNum. StoresNum. StorefilesStorefile Size UncompressedStorefile SizeIndex SizeBloom Size
<& serverNameLink; serverName=serverName; &><% String.format("%,d", storeCount) %><% String.format("%,d", storeFileCount) %><% storeUncompressedSizeMBStr %><% storeFileSizeMBStr %><% totalStaticIndexSizeKBStr %><% totalStaticBloomSizeKBStr %>
- - -<%def compactionStats> -<%args> - ServerName [] serverNames; - - - - - - - - - - - - -<%java> -for (ServerName serverName: serverNames) { - -ServerMetrics sl = master.getServerManager().getLoad(serverName); -if (sl != null) { -long totalCompactingCells = 0; -long totalCompactedCells = 0; -for (RegionMetrics rl : sl.getRegionMetrics().values()) { - totalCompactingCells += rl.getCompactingCellCount(); - totalCompactedCells += rl.getCompactedCellCount(); -} -String percentDone = ""; -if (totalCompactingCells > 0) { - percentDone = String.format("%.2f", 100 * - ((float) totalCompactedCells / totalCompactingCells)) + "%"; -} - - - - - - - - -<%java> - } else { - -<& emptyStat; serverName=serverName; &> -<%java> - } -} - - -
ServerNameNum. Compacting CellsNum. Compacted CellsRemaining CellsCompaction Progress
<& serverNameLink; serverName=serverName; &><% String.format("%,d", totalCompactingCells) %><% String.format("%,d", totalCompactedCells) %><% String.format("%,d", totalCompactingCells - totalCompactedCells) %><% percentDone %>
- - -<%def replicationStats> -<%args> - ServerName [] serverNames; - -<%java> - HashMap>> replicationLoadSourceMap - = master.getReplicationLoad(serverNames); - List peers = null; - if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0){ - peers = new ArrayList<>(replicationLoadSourceMap.keySet()); - Collections.sort(peers); - } - - -<%if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0) %> - -
-
    - <%java> - String active = "active"; - for (String peer : peers){ - -
  • >Peer <% peer %>
    • - <%java> - active = ""; - } - -
-
- <%java> - active = "active"; - for (String peer : peers){ - -
- - - - - - - - - <%for Pair pair: replicationLoadSourceMap.get(peer) %> - - - - - - - -
ServerAgeOfLastShippedOpSizeOfLogQueueReplicationLag
<& serverNameLink; serverName=pair.getFirst(); &><% StringUtils.humanTimeDiff(pair.getSecond().getAgeOfLastShippedOp()) %><% pair.getSecond().getSizeOfLogQueue() %><% pair.getSecond().getReplicationLag() == Long.MAX_VALUE ? "UNKNOWN" : StringUtils.humanTimeDiff(pair.getSecond().getReplicationLag()) %>
-
- <%java> - active = ""; - } - -
-

If the replication delay is UNKNOWN, that means this walGroup doesn't start replicate yet and it may get disabled.

-
-<%else> -

No Peers Metrics

- - - - - -<%def serverNameLink> - <%args> - ServerName serverName; - - <%java> - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; - - - <%if infoPort > 0%> - <% serverName.getServerName() %> - <%else> - <% serverName.getServerName() %> - - - -<%def emptyStat> - <%args> - ServerName serverName; - - - <& serverNameLink; serverName=serverName; &> - - - - - - - - - - - - - - - diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon deleted file mode 100644 index 9a98cfefed7f..000000000000 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon +++ /dev/null @@ -1,119 +0,0 @@ -<%doc> - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - - - - - -
- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 4b8047b9e446..15426675deca 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -243,7 +243,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; <%else> <%java> String host = masterServerName.getHostname() + ":" + infoPort; - String url = "//" + host + "/master-status"; + String url = "//" + host + "/master.jsp"; <% host %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusConstants.java new file mode 100644 index 000000000000..7432e529dfc8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusConstants.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.http; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Constants used by the web UI JSP pages. + */ +@InterfaceAudience.Private +public final class MasterStatusConstants { + + public static final String FRAGS = "frags"; + public static final String SERVER_NAMES = "serverNames"; + public static final String SERVER_NAME = "serverName"; + public static final String RS_GROUP_INFOS = "rsGroupInfos"; + public static final String COLLECT_SERVERS = "collectServers"; + public static final String FILTER = "filter"; + public static final String FORMAT = "format"; + public static final String PARENT = "parent"; + + private MasterStatusConstants() { + // Do not instantiate. + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusServlet.java index 09bb5375a5d5..564e5f01124b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusServlet.java @@ -18,25 +18,13 @@ package org.apache.hadoop.hbase.master.http; import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.RegionInfoBuilder; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.assignment.RegionStateNode; -import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.yetus.audience.InterfaceAudience; /** - * The servlet responsible for rendering the index page of the master. + * Only kept for redirecting to master.jsp. */ @InterfaceAudience.Private public class MasterStatusServlet extends HttpServlet { @@ -44,52 +32,6 @@ public class MasterStatusServlet extends HttpServlet { @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); - assert master != null : "No Master in context!"; - - response.setContentType("text/html"); - - Configuration conf = master.getConfiguration(); - - Map frags = getFragmentationInfo(master, conf); - ServerName metaLocation = null; - List servers = null; - Set deadServers = null; - - if (master.isActiveMaster()) { - metaLocation = getMetaLocationOrNull(master); - ServerManager serverManager = master.getServerManager(); - if (serverManager != null) { - deadServers = serverManager.getDeadServers().copyServerNames(); - servers = serverManager.getOnlineServersList(); - } - } - - MasterStatusTmpl tmpl = - new MasterStatusTmpl().setFrags(frags).setMetaLocation(metaLocation).setServers(servers) - .setDeadServers(deadServers).setCatalogJanitorEnabled(master.isCatalogJanitorEnabled()); - - if (request.getParameter("filter") != null) tmpl.setFilter(request.getParameter("filter")); - if (request.getParameter("format") != null) tmpl.setFormat(request.getParameter("format")); - tmpl.render(response.getWriter(), master); - } - - private ServerName getMetaLocationOrNull(HMaster master) { - RegionStateNode rsn = master.getAssignmentManager().getRegionStates() - .getRegionStateNode(RegionInfoBuilder.FIRST_META_REGIONINFO); - if (rsn != null) { - return rsn.isInState(RegionState.State.OPEN) ? rsn.getRegionLocation() : null; - } - return null; - } - - private Map getFragmentationInfo(HMaster master, Configuration conf) - throws IOException { - boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); - if (showFragmentation) { - return FSUtils.getTableFragmentation(master); - } else { - return null; - } + response.sendRedirect(request.getContextPath() + "/master.jsp"); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusUtil.java new file mode 100644 index 000000000000..221e43f8e114 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterStatusUtil.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.http; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.assignment.RegionStateNode; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Utility used by the web UI JSP pages. + */ +@InterfaceAudience.Private +public final class MasterStatusUtil { + + private MasterStatusUtil() { + // Do not instantiate. + } + + public static String getUserTables(HMaster master, List tables) { + if (master.isInitialized()) { + try { + Map descriptorMap = master.getTableDescriptors().getAll(); + if (descriptorMap != null) { + for (TableDescriptor desc : descriptorMap.values()) { + if (!desc.getTableName().isSystemTable()) { + tables.add(desc); + } + } + } + } catch (IOException e) { + return "Got user tables error, " + e.getMessage(); + } + } + return null; + } + + public static Map getFragmentationInfo(HMaster master, Configuration conf) + throws IOException { + boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); + if (showFragmentation) { + return FSUtils.getTableFragmentation(master); + } else { + return null; + } + } + + public static ServerName getMetaLocationOrNull(HMaster master) { + RegionStateNode rsn = master.getAssignmentManager().getRegionStates() + .getRegionStateNode(RegionInfoBuilder.FIRST_META_REGIONINFO); + if (rsn != null) { + return rsn.isInState(RegionState.State.OPEN) ? rsn.getRegionLocation() : null; + } + return null; + } + + public static String serverNameLink(HMaster master, ServerName serverName) { + int infoPort = master.getRegionServerInfoPort(serverName); + String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + if (infoPort > 0) { + return "" + serverName.getServerName() + ""; + } else { + return serverName.getServerName(); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/RegionVisualizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/RegionVisualizer.java index 09171c3a8c2f..ceb08bcedf2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/RegionVisualizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/RegionVisualizer.java @@ -58,7 +58,7 @@ /** * Support class for the "Region Visualizer" rendered out of - * {@code src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionVisualizerTmpl.jamon} + * {@code src/main/resources/hbase-webapps/master/regionVisualizer.jsp} */ @InterfaceAudience.Private public class RegionVisualizer extends AbstractHBaseTool { diff --git a/hbase-server/src/main/resources/hbase-webapps/master/assignmentManagerStatus.jsp b/hbase-server/src/main/resources/hbase-webapps/master/assignmentManagerStatus.jsp new file mode 100644 index 000000000000..0966d04316b4 --- /dev/null +++ b/hbase-server/src/main/resources/hbase-webapps/master/assignmentManagerStatus.jsp @@ -0,0 +1,117 @@ +<%-- +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.quotas.QuotaUtil" + import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer" + import="org.apache.hadoop.hbase.master.assignment.AssignmentManager" + import="org.apache.hadoop.hbase.master.RegionState" + import="java.util.SortedSet" + import="org.apache.hadoop.hbase.master.assignment.RegionStates" + import="org.apache.hadoop.hbase.client.RegionInfoDisplay" %> +<% + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + AssignmentManager assignmentManager = master.getAssignmentManager(); + int limit = 100; + + SortedSet rit = assignmentManager.getRegionStates().getRegionsInTransitionOrderedByTimestamp(); + +if (!rit.isEmpty()) { + long currentTime = System.currentTimeMillis(); + AssignmentManager.RegionInTransitionStat ritStat = assignmentManager.computeRegionInTransitionStat(); + + int numOfRITs = rit.size(); + int ritsPerPage = Math.min(5, numOfRITs); + int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage); +%> +
+

Regions in Transition

+

<%= numOfRITs %> region(s) in transition. + <% if(ritStat.hasRegionsTwiceOverThreshold()) { %> + + <% } else if ( ritStat.hasRegionsOverThreshold()) { %> + + <% } else { %> + + <% } %> + <%= ritStat.getTotalRITsOverThreshold() %> region(s) in transition for + more than <%= ritStat.getRITThreshold() %> milliseconds. + +

+
+
+ <% int recordItr = 0; %> + <% for (RegionState rs : rit) { %> + <% if((recordItr % ritsPerPage) == 0 ) { %> + <% if(recordItr == 0) { %> +
+ <% } else { %> +
+ <% } %> + + + <% } %> + + <% if(ritStat.isRegionTwiceOverThreshold(rs.getRegion())) { %> + + <% } else if ( ritStat.isRegionOverThreshold(rs.getRegion())) { %> + + <% } else { %> + + <% } %> + <% + String retryStatus = "0"; + RegionStates.RegionFailedOpen regionFailedOpen = assignmentManager + .getRegionStates().getFailedOpen(rs.getRegion()); + if (regionFailedOpen != null) { + retryStatus = Integer.toString(regionFailedOpen.getRetries()); + } else if (rs.getState() == RegionState.State.FAILED_OPEN) { + retryStatus = "Failed"; + } + %> + + + + + <% recordItr++; %> + <% if((recordItr % ritsPerPage) == 0) { %> +
RegionStateRIT time (ms) Retries
<%= rs.getRegion().getEncodedName() %> + <%= RegionInfoDisplay.getDescriptiveNameFromRegionStateForDisplay(rs, + assignmentManager.getConfiguration()) %><%= (currentTime - rs.getStamp()) %> <%= retryStatus %>
+
+ <% } %> + <% } %> + + <% if((recordItr % ritsPerPage) != 0) { %> + <% for (; (recordItr % ritsPerPage) != 0 ; recordItr++) { %> + + <% } %> + +
+ <% } %> +
+ + + +
+
+<% } %> + diff --git a/hbase-server/src/main/resources/hbase-webapps/master/backupMasterStatus.jsp b/hbase-server/src/main/resources/hbase-webapps/master/backupMasterStatus.jsp new file mode 100644 index 000000000000..cada34472c95 --- /dev/null +++ b/hbase-server/src/main/resources/hbase-webapps/master/backupMasterStatus.jsp @@ -0,0 +1,66 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.*" + import="org.apache.hadoop.hbase.ServerName" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hbase.thirdparty.com.google.common.base.Preconditions" %> +<% + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + if (!master.isActiveMaster()) { + + ServerName active_master = master.getActiveMaster().orElse(null); + Preconditions.checkState(active_master != null, "Failed to retrieve active master's ServerName!"); + int activeInfoPort = master.getActiveMasterInfoPort(); +%> +
+
+

Backup Master <%= master.getServerName().getHostname() %>

+
+
+

Current Active Master: <%= active_master.getHostname() %>

+ <% } else { %> +

Backup Masters

+ + + + + + + + <% + Collection backup_masters = master.getBackupMasters(); + ServerName [] backupServerNames = backup_masters.toArray(new ServerName[backup_masters.size()]); + Arrays.sort(backupServerNames); + for (ServerName serverName : backupServerNames) { + int infoPort = master.getBackupMasterInfoPort(serverName); + %> + + + + + + <% } %> + +
ServerNamePortStart Time
<%= serverName.getHostname() %> + <%= serverName.getPort() %><%= new Date(serverName.getStartCode()) %>
Total:<%= backupServerNames.length %>
+<% } %> diff --git a/hbase-server/src/main/resources/hbase-webapps/master/catalogTables.jsp b/hbase-server/src/main/resources/hbase-webapps/master/catalogTables.jsp new file mode 100644 index 000000000000..5bc67e0b79ed --- /dev/null +++ b/hbase-server/src/main/resources/hbase-webapps/master/catalogTables.jsp @@ -0,0 +1,84 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> + +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.*" + import="org.apache.hadoop.hbase.NamespaceDescriptor" + import="org.apache.hadoop.hbase.TableName" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.quotas.QuotaUtil" + import="org.apache.hadoop.hbase.security.access.PermissionStorage" + import="org.apache.hadoop.hbase.security.visibility.VisibilityConstants" + import="org.apache.hadoop.hbase.tool.CanaryTool" + import="org.apache.hadoop.hbase.client.*" + import="org.apache.hadoop.hbase.master.http.MasterStatusConstants" %> + +<% + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + + Map frags = (Map) request.getAttribute(MasterStatusConstants.FRAGS); + + List sysTables = master.isInitialized() ? + master.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR) : null; +%> + +<%if (sysTables != null && sysTables.size() > 0) { %> + + + + <% if (frags != null) { %> + + <% } %> + + + <% for (TableDescriptor systemTable : sysTables) { %> + + <% TableName tableName = systemTable.getTableName();%> + + <% if (frags != null) { %> + + <% } %> + <% String description = null; + if (tableName.equals(TableName.META_TABLE_NAME)){ + description = "The hbase:meta table holds references to all User Table regions."; + } else if (tableName.equals(CanaryTool.DEFAULT_WRITE_TABLE_NAME)){ + description = "The hbase:canary table is used to sniff the write availability of" + + " each regionserver."; + } else if (tableName.equals(PermissionStorage.ACL_TABLE_NAME)){ + description = "The hbase:acl table holds information about acl."; + } else if (tableName.equals(VisibilityConstants.LABELS_TABLE_NAME)){ + description = "The hbase:labels table holds information about visibility labels."; + } else if (tableName.equals(TableName.NAMESPACE_TABLE_NAME)){ + description = "The hbase:namespace table holds information about namespaces."; + } else if (tableName.equals(QuotaUtil.QUOTA_TABLE_NAME)){ + description = "The hbase:quota table holds quota information about number" + + " or size of requests in a given time frame."; + } else if (tableName.equals(TableName.valueOf("hbase:rsgroup"))){ + description = "The hbase:rsgroup table holds information about regionserver groups."; + } else if (tableName.equals(TableName.valueOf("hbase:replication"))) { + description = "The hbase:replication table tracks cross cluster replication through " + + "WAL file offsets."; + } + %> + + + <% } %> +
Table NameFrag.Description
<%= tableName %><%= frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()) + "%" : "n/a" %><%= description %>
+<% } %> diff --git a/hbase-server/src/main/resources/hbase-webapps/master/deadRegionServers.jsp b/hbase-server/src/main/resources/hbase-webapps/master/deadRegionServers.jsp new file mode 100644 index 000000000000..daef9a58eca9 --- /dev/null +++ b/hbase-server/src/main/resources/hbase-webapps/master/deadRegionServers.jsp @@ -0,0 +1,96 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="org.apache.hadoop.hbase.ServerName" + import="java.util.*" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.master.DeadServer" + import="org.apache.hadoop.hbase.rsgroup.RSGroupInfo" + import="org.apache.hadoop.hbase.master.ServerManager" + import="org.apache.hadoop.hbase.RSGroupTableAccessor" %> +<% + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + + ServerManager serverManager = master.getServerManager(); + + Set deadServers = null; + + if (master.isActiveMaster()) { + if (serverManager != null) { + deadServers = serverManager.getDeadServers().copyServerNames(); + } + } +%> + +<% if (deadServers != null && deadServers.size() > 0) { %> +

Dead Region Servers

+ + + + + + <% if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null + && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null + && master.getServerManager().getOnlineServersList().size() > 0) { %> + + <% } %> + +<% + List groups = null; + DeadServer deadServerUtil = master.getServerManager().getDeadServers(); + if (!master.isInMaintenanceMode() && master.getMasterCoprocessorHost() != null + && master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null + && master.getServerManager().getOnlineServersList().size() > 0) { + groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection()); + } + ServerName [] deadServerNames = deadServers.toArray(new ServerName[deadServers.size()]); + Arrays.sort(deadServerNames); + for (ServerName deadServerName: deadServerNames) { + String rsGroupName = null; + if (groups != null){ + for (RSGroupInfo rsGroupInfo : groups) { + if (rsGroupInfo.containsServer(deadServerName.getAddress())) { + rsGroupName = rsGroupInfo.getName(); + break; + } + } + if (rsGroupName == null) { + rsGroupName = RSGroupInfo.DEFAULT_GROUP; + } + } + %> + + + + + <% if (rsGroupName != null) { %> + + <% } %> + + <% + } + %> + + + + + +
ServerNameStop timeRSGroup
<%= deadServerName %><%= deadServerUtil.getTimeOfDeath(deadServerName) %><%= rsGroupName %>
Total: servers: <%= deadServers.size() %>
+<% } %> diff --git a/hbase-server/src/main/resources/hbase-webapps/master/header.jsp b/hbase-server/src/main/resources/hbase-webapps/master/header.jsp index 267658d1ca0a..0c03b36e6f2b 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/header.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/header.jsp @@ -42,13 +42,13 @@