From 7a77f9495bb7a4cf6ae1ca1b9202a6b20a1a0947 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Mon, 7 Sep 2026 10:16:06 +0200 Subject: [PATCH 1/2] Use JDK intrinsics for Binary equality/comparison/hashing Replace the hand-rolled scalar byte loops in Binary#equals, Binary#lexicographicCompare, and Binary#hashCode with the JDK 9+ Arrays.equals(byte[],int,int,byte[],int,int), Arrays.compareUnsigned(byte[],int,int,byte[],int,int), Arrays.hashCode, and ByteBuffer.mismatch APIs. These range overloads route through ArraysSupport.vectorizedMismatch, an @IntrinsicCandidate helper HotSpot substitutes with a SIMD byte-scan (SSE / AVX2 / NEON on modern hardware). Measured on this project's BinaryComparisonBenchmark (JDK 17, 1 fork, 3x1s warmup, 5x1s measurement, throughput): equalsMatch_bytesBytes len=512 9.1M -> 39.1M ops/s (4.30x) equalsMatch_bytesBytes len= 64 41.4M -> 160.9M ops/s (3.89x) equalsMismatch_bytesBytes len=512 14.6M -> 55.5M ops/s (3.79x) compareTo_bytesBytes len=512 15.1M -> 58.8M ops/s (3.91x) compareTo_bytesBytes len= 64 59.0M -> 199.4M ops/s (3.38x) equalsMismatch_bytesBuf len= 64 63.8M -> 190.2M ops/s (2.98x) Short values (len=8) see only 1.1-1.7x because there's barely enough work for one SIMD lane. hashCode is separate. The 31*h+b polynomial has a serial dependency across iterations, so SIMD needs a lane-split algebraic trick that HotSpot only gained in JDK 21 (via ArraysSupport.vectorizedHashCode, which is @IntrinsicCandidate). On JDK 17 -- this project's target -- Arrays.hashCode is a plain scalar loop and shows no measured speedup here. The call is kept for forward compatibility: JDK 21+ runtimes pick up the vectorized intrinsic silently; a bespoke loop would stay stuck at scalar. These primitives sit on Binary equality, comparison and hash-map probing, which is the hot code path for statistics min/max maintenance, dictionary probing, predicate evaluation, and bloom-filter build -- i.e. broadly amortized across reader and writer. Semantics are preserved bit-for-bit: - hashCode(byte[]) uses the same 31*h + b polynomial (Arrays.hashCode on full arrays; the polynomial expanded for slices). - equals is bytewise identity. - lexicographicCompare is unsigned bytewise with shorter-first tie-break on prefix match, matching Arrays.compareUnsigned's contract. The change is JIT-friendly across the four Binary/Binary shape pairs (byte[]/byte[], byte[]/ByteBuffer, ByteBuffer/ByteBuffer) and for heap-backed ByteBuffers unwraps to the intrinsic byte[] path via buffer.array(). Follows the same technique already accepted for DeltaByteArrayWriter in PR apache/parquet-java#3465. Complementary to the Binary hashCode caching in the open dictionary-optimization PR apache/parquet-java#3566: caching reduces call frequency; intrinsics speed up the calls that remain plus equals/compareTo which caching does not address. Includes a JMH micro-benchmark (parquet-benchmarks/BinaryComparisonBenchmark) covering equals / compareTo / hashCode across length regimes 8, 64 and 512 for the three Binary shape combinations, with both worst-case (full match) and average-case (mid-length mismatch) inputs. --- .../benchmarks/BinaryComparisonBenchmark.java | 199 ++++++++++++++++++ .../org/apache/parquet/io/api/Binary.java | 174 ++++++++------- 2 files changed, 302 insertions(+), 71 deletions(-) create mode 100644 parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/BinaryComparisonBenchmark.java diff --git a/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/BinaryComparisonBenchmark.java b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/BinaryComparisonBenchmark.java new file mode 100644 index 0000000000..c035a27eb3 --- /dev/null +++ b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/BinaryComparisonBenchmark.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.benchmarks; + +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.parquet.io.api.Binary; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Micro-benchmarks for {@link Binary#equals}, {@link Binary#compareTo}, and + * {@link Binary#hashCode()}. These sit on statistics min/max maintenance, + * dictionary hash-map probing, predicate evaluation, and bloom-filter build, + * so improvements are broadly amortized across reader and writer paths. + * + *

Each invocation performs {@value #PAIRS} comparisons. The + * {@code length} parameter covers three regimes: + *

+ * + *

{@code equalsMatch} compares equal binaries (the worst case: the + * mismatch scan walks the full length). {@code equalsMismatch} places the + * differing byte at length/2 (average case). {@code compareTo} uses the same + * mid-mismatch data. + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Fork(1) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@State(Scope.Thread) +public class BinaryComparisonBenchmark { + + /** Number of paired comparisons per @Benchmark invocation. */ + static final int PAIRS = 10_000; + + @Param({"8", "64", "512"}) + public int length; + + // Heap-array backed + private Binary[] leftArr; + private Binary[] rightEqualArr; + private Binary[] rightMidMismatchArr; + + // ByteBuffer-backed (heap) + private Binary[] leftBuf; + private Binary[] rightEqualBuf; + private Binary[] rightMidMismatchBuf; + + @Setup(Level.Trial) + public void setup() { + Random r = new Random(TestDataFactory.DEFAULT_SEED); + leftArr = new Binary[PAIRS]; + rightEqualArr = new Binary[PAIRS]; + rightMidMismatchArr = new Binary[PAIRS]; + leftBuf = new Binary[PAIRS]; + rightEqualBuf = new Binary[PAIRS]; + rightMidMismatchBuf = new Binary[PAIRS]; + + for (int i = 0; i < PAIRS; i++) { + byte[] a = new byte[length]; + r.nextBytes(a); + byte[] aCopy = a.clone(); + byte[] aMidMismatch = a.clone(); + // Flip a bit at position length/2 to force a mid-length mismatch. + aMidMismatch[length / 2] ^= (byte) 0xFF; + + leftArr[i] = Binary.fromConstantByteArray(a); + rightEqualArr[i] = Binary.fromConstantByteArray(aCopy); + rightMidMismatchArr[i] = Binary.fromConstantByteArray(aMidMismatch); + + leftBuf[i] = Binary.fromConstantByteBuffer(ByteBuffer.wrap(a.clone())); + rightEqualBuf[i] = Binary.fromConstantByteBuffer(ByteBuffer.wrap(aCopy.clone())); + rightMidMismatchBuf[i] = Binary.fromConstantByteBuffer(ByteBuffer.wrap(aMidMismatch.clone())); + } + } + + // ---- byte[] vs byte[] ---- + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void equalsMatch_bytesBytes(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].equals(rightEqualArr[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void equalsMismatch_bytesBytes(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].equals(rightMidMismatchArr[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void compareTo_bytesBytes(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].compareTo(rightMidMismatchArr[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void hashCode_bytes(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].hashCode()); + } + } + + // ---- ByteBuffer vs ByteBuffer (heap-backed) ---- + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void equalsMatch_bufBuf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftBuf[i].equals(rightEqualBuf[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void equalsMismatch_bufBuf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftBuf[i].equals(rightMidMismatchBuf[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void compareTo_bufBuf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftBuf[i].compareTo(rightMidMismatchBuf[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void hashCode_buf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftBuf[i].hashCode()); + } + } + + // ---- Mixed: byte[] vs ByteBuffer (predicate pushdown pattern) ---- + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void equalsMismatch_bytesBuf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].equals(rightMidMismatchBuf[i])); + } + } + + @Benchmark + @OperationsPerInvocation(PAIRS) + public void compareTo_bytesBuf(Blackhole bh) { + for (int i = 0; i < PAIRS; i++) { + bh.consume(leftArr[i].compareTo(rightMidMismatchBuf[i])); + } + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/io/api/Binary.java b/parquet-column/src/main/java/org/apache/parquet/io/api/Binary.java index f53199803b..f480aec6c2 100644 --- a/parquet-column/src/main/java/org/apache/parquet/io/api/Binary.java +++ b/parquet-column/src/main/java/org/apache/parquet/io/api/Binary.java @@ -654,27 +654,62 @@ public static int lexicographicCompare(Binary one, Binary other) { return one.lexicographicCompare(other); } - /** - * @param array - * @param offset - * @param length - * @return - * @see {@link Arrays#hashCode(byte[])} - */ + // --------------------------------------------------------------------------- + // Byte comparison / hashing primitives. + // + // equals / lexicographicCompare / mismatch delegate to Arrays.* / ByteBuffer.mismatch + // range overloads, which route through ArraysSupport.vectorizedMismatch -- + // an @IntrinsicCandidate helper HotSpot substitutes with a SIMD byte-scan + // (SSE / AVX2 / NEON on modern hardware). Compared to the previous hand-rolled + // scalar byte loops, measured throughput on this project's BinaryComparisonBenchmark + // (JDK 17) is roughly 3-4x for 64- and 512-byte values and ~1.1-1.7x for 8-byte + // values, where the intrinsic barely wins. These primitives sit on statistics + // min/max maintenance, dictionary hash-map probing, predicate evaluation, and + // bloom-filter build, so the win is broad. + // + // hashCode is a different story. The 31*h+b polynomial has a serial dependency + // across iterations, so SIMD needs a lane-split algebraic trick. HotSpot only + // gained that intrinsic in JDK 21 (via ArraysSupport.vectorizedHashCode, which + // is @IntrinsicCandidate). On JDK 17 -- this project's target -- Arrays.hashCode + // is a plain scalar loop and delivers no measured speedup over the hand-rolled + // version it replaces. The call is kept for forward compatibility: JDK 21+ + // runtimes pick up the vectorized intrinsic silently, whereas a bespoke loop + // would stay stuck at scalar forever. + // + // Semantics are preserved bit-for-bit: + // - hashCode(byte[]) uses the same 31*h + b polynomial as before + // (that's what the JDK's own Arrays.hashCode computes). + // - equals is bytewise identity. + // - lexicographicCompare is unsigned bytewise, with shorter-runs-first + // tie-break on prefix match, matching Arrays.compareUnsigned. + // --------------------------------------------------------------------------- + private static final int hashCode(byte[] array, int offset, int length) { + // Route full-array hashCode through Arrays.hashCode. On JDK 17 that is a + // plain scalar loop; on JDK 21+ it delegates to the vectorized intrinsic + // ArraysSupport.vectorizedHashCode. The slice path below reproduces the + // same 31*h+b polynomial exactly. + if (offset == 0 && length == array.length) { + return Arrays.hashCode(array); + } int result = 1; - for (int i = offset; i < offset + length; i++) { - byte b = array[i]; - result = 31 * result + b; + int end = offset + length; + for (int i = offset; i < end; i++) { + result = 31 * result + array[i]; } return result; } private static final int hashCode(ByteBuffer buf, int offset, int length) { + // If the buffer is heap-backed, fall through to the byte[] path so JDK 21+ + // can pick up the vectorized hashCode intrinsic. + if (buf.hasArray()) { + return hashCode(buf.array(), buf.arrayOffset() + offset, length); + } int result = 1; - for (int i = offset; i < offset + length; i++) { - byte b = buf.get(i); - result = 31 * result + b; + int end = offset + length; + for (int i = offset; i < end; i++) { + result = 31 * result + buf.get(i); } return result; } @@ -684,12 +719,18 @@ private static final boolean equals( if (buf1 == null && buf2 == null) return true; if (buf1 == null || buf2 == null) return false; if (length1 != length2) return false; - for (int i = 0; i < length1; i++) { - if (buf1.get(i + offset1) != buf2.get(i + offset2)) { - return false; - } + // Fast-path both heap-backed: use vectorized Arrays.equals on the underlying arrays. + if (buf1.hasArray() && buf2.hasArray()) { + final int o1 = buf1.arrayOffset() + offset1; + final int o2 = buf2.arrayOffset() + offset2; + return Arrays.equals(buf1.array(), o1, o1 + length1, buf2.array(), o2, o2 + length2); } - return true; + // ByteBuffer.mismatch is intrinsified on JDK 11+ and delegates to + // ArraysSupport.vectorizedMismatch; use it via sliced views so it applies + // to non-array-backed (direct / read-only) buffers as well. + ByteBuffer s1 = slice(buf1, offset1, length1); + ByteBuffer s2 = slice(buf2, offset2, length2); + return s1.mismatch(s2) < 0; } private static final boolean equals( @@ -697,6 +738,10 @@ private static final boolean equals( if (array1 == null && buf == null) return true; if (array1 == null || buf == null) return false; if (length1 != length2) return false; + if (buf.hasArray()) { + final int o2 = buf.arrayOffset() + offset2; + return Arrays.equals(array1, offset1, offset1 + length1, buf.array(), o2, o2 + length2); + } for (int i = 0; i < length1; i++) { if (array1[i + offset1] != buf.get(i + offset2)) { return false; @@ -706,82 +751,69 @@ private static final boolean equals( } /** - * @param array1 - * @param offset1 - * @param length1 - * @param array2 - * @param offset2 - * @param length2 - * @return - * @see {@link Arrays#equals(byte[], byte[])} + * @see Arrays#equals(byte[], int, int, byte[], int, int) */ private static final boolean equals( byte[] array1, int offset1, int length1, byte[] array2, int offset2, int length2) { if (array1 == null && array2 == null) return true; if (array1 == null || array2 == null) return false; - if (length1 != length2) return false; - if (array1 == array2 && offset1 == offset2) return true; - for (int i = 0; i < length1; i++) { - if (array1[i + offset1] != array2[i + offset2]) { - return false; - } - } - return true; + // Arrays.equals(byte[], int, int, byte[], int, int) delegates to + // ArraysSupport.mismatch, which routes through the @IntrinsicCandidate + // vectorizedMismatch helper HotSpot substitutes with a SIMD byte-scan. + return Arrays.equals(array1, offset1, offset1 + length1, array2, offset2, offset2 + length2); } private static final int lexicographicCompare( byte[] array1, int offset1, int length1, byte[] array2, int offset2, int length2) { if (array1 == null && array2 == null) return 0; if (array1 == null || array2 == null) return array1 != null ? 1 : -1; - - int minLen = Math.min(length1, length2); - for (int i = 0; i < minLen; i++) { - int res = unsignedCompare(array1[i + offset1], array2[i + offset2]); - if (res != 0) { - return res; - } - } - - return length1 - length2; + // Arrays.compareUnsigned routes through the same ArraysSupport.vectorizedMismatch + // intrinsic: a SIMD mismatch scan, then an unsigned compare on the mismatching + // pair, with shorter-first tie-break on a full prefix match -- semantically + // identical to the previous hand-rolled loop. + return Arrays.compareUnsigned(array1, offset1, offset1 + length1, array2, offset2, offset2 + length2); } private static final int lexicographicCompare( byte[] array, int offset1, int length1, ByteBuffer buffer, int offset2, int length2) { if (array == null && buffer == null) return 0; if (array == null || buffer == null) return array != null ? 1 : -1; - - int minLen = Math.min(length1, length2); - for (int i = 0; i < minLen; i++) { - int res = unsignedCompare(array[i + offset1], buffer.get(i + offset2)); - if (res != 0) { - return res; - } + if (buffer.hasArray()) { + final int o2 = buffer.arrayOffset() + offset2; + return Arrays.compareUnsigned(array, offset1, offset1 + length1, buffer.array(), o2, o2 + length2); } - - return length1 - length2; + // Compare via ByteBuffer.mismatch to find the first differing position. + ByteBuffer left = ByteBuffer.wrap(array, offset1, length1).slice(); + ByteBuffer right = slice(buffer, offset2, length2); + int mm = left.mismatch(right); + if (mm < 0) return length1 - length2; + if (mm >= length1) return -1; + if (mm >= length2) return 1; + return (array[offset1 + mm] & 0xFF) - (buffer.get(offset2 + mm) & 0xFF); } private static final int lexicographicCompare( ByteBuffer buffer1, int offset1, int length1, ByteBuffer buffer2, int offset2, int length2) { if (buffer1 == null && buffer2 == null) return 0; if (buffer1 == null || buffer2 == null) return buffer1 != null ? 1 : -1; - - int minLen = Math.min(length1, length2); - for (int i = 0; i < minLen; i++) { - int res = unsignedCompare(buffer1.get(i + offset1), buffer2.get(i + offset2)); - if (res != 0) { - return res; - } - } - - return length1 - length2; - } - - private static int unsignedCompare(byte b1, byte b2) { - return toUnsigned(b1) - toUnsigned(b2); - } - - private static final int toUnsigned(byte b) { - return b & 0xFF; + if (buffer1.hasArray() && buffer2.hasArray()) { + final int o1 = buffer1.arrayOffset() + offset1; + final int o2 = buffer2.arrayOffset() + offset2; + return Arrays.compareUnsigned(buffer1.array(), o1, o1 + length1, buffer2.array(), o2, o2 + length2); + } + ByteBuffer left = slice(buffer1, offset1, length1); + ByteBuffer right = slice(buffer2, offset2, length2); + int mm = left.mismatch(right); + if (mm < 0) return length1 - length2; + if (mm >= length1) return -1; + if (mm >= length2) return 1; + return (buffer1.get(offset1 + mm) & 0xFF) - (buffer2.get(offset2 + mm) & 0xFF); + } + + /** Return a sliced view of {@code buf} covering {@code [offset, offset+length)} without mutating {@code buf}. */ + private static ByteBuffer slice(ByteBuffer buf, int offset, int length) { + ByteBuffer dup = buf.duplicate(); + dup.position(offset).limit(offset + length); + return dup.slice(); } } From ce1006b67eda198260edb09de309a06ce62216fd Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 8 Sep 2026 09:09:30 +0200 Subject: [PATCH 2/2] Binary test coverage --- .../org/apache/parquet/io/api/TestBinary.java | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/parquet-column/src/test/java/org/apache/parquet/io/api/TestBinary.java b/parquet-column/src/test/java/org/apache/parquet/io/api/TestBinary.java index d925572bca..80b6816719 100644 --- a/parquet-column/src/test/java/org/apache/parquet/io/api/TestBinary.java +++ b/parquet-column/src/test/java/org/apache/parquet/io/api/TestBinary.java @@ -30,6 +30,7 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Random; import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.TestBinary.BinaryFactory.BinaryAndOriginal; import org.junit.jupiter.api.Test; @@ -490,4 +491,168 @@ public void testFromCharSequenceRejectsMalformedUtf16() { .isInstanceOf(ParquetEncodingException.class) .hasMessage("Failed to encode CharSequence as UTF-8."); } + + /* + * hashCode / equals / compareTo: guard the intrinsic-backed impls against + * accidental drift. Assertions pin each result to an independent scalar + * oracle or a pre-computed literal — any refactor that changes the bit-for- + * bit output on any shape (byte[], slice, heap ByteBuffer, direct ByteBuffer) + * fails these tests immediately. + */ + + @Test + public void testHashCodeMatchesScalarOracleAcrossShapesAndLengths() { + // Covers unrolled and remainder paths of Arrays.hashCode / the byte-loop fallback. + Random rnd = new Random(0xC0FFEEL); + int[] lengths = {0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 511, 512, 1023, 1024}; + for (int len : lengths) { + byte[] bytes = new byte[len]; + rnd.nextBytes(bytes); + int expected = scalarHashCode(bytes, 0, len); + + // Shape 1: constant byte[] (full). + assertThat(Binary.fromConstantByteArray(bytes).hashCode()) + .as("byte[] len=%d", len) + .isEqualTo(expected); + + // Shape 2: sliced byte[] with non-zero offset (exercises the slice branch). + byte[] padded = new byte[len + 17]; + System.arraycopy(bytes, 0, padded, 5, len); + assertThat(Binary.fromConstantByteArray(padded, 5, len).hashCode()) + .as("slice len=%d", len) + .isEqualTo(expected); + + // Shape 3: heap ByteBuffer (unwraps to byte[] via array()). + assertThat(Binary.fromConstantByteBuffer(ByteBuffer.wrap(bytes)).hashCode()) + .as("heapBB len=%d", len) + .isEqualTo(expected); + + // Shape 4: direct ByteBuffer (exercises the buf.get(i) fallback path). + ByteBuffer direct = ByteBuffer.allocateDirect(Math.max(len, 1)); + direct.limit(len); + direct.put(bytes).flip(); + assertThat(Binary.fromConstantByteBuffer(direct).hashCode()) + .as("directBB len=%d", len) + .isEqualTo(expected); + } + } + + @Test + public void testHashCodeGoldenValues() { + // Independent 31*h+b calculation with initial h=1 — pins the polynomial itself + // (initial value, sign extension, order of ops), catching drifts a scalar + // oracle would silently reproduce. + assertThat(Binary.fromConstantByteArray(new byte[0]).hashCode()).isEqualTo(1); + assertThat(Binary.fromConstantByteArray(new byte[] {0}).hashCode()).isEqualTo(31); + assertThat(Binary.fromConstantByteArray(new byte[] {1}).hashCode()).isEqualTo(32); + // Sign extension: (byte) -1 must contribute -1 (not 255) to the polynomial. + assertThat(Binary.fromConstantByteArray(new byte[] {-1}).hashCode()).isEqualTo(30); + assertThat(Binary.fromConstantByteArray(new byte[] {127}).hashCode()).isEqualTo(158); + assertThat(Binary.fromConstantByteArray(new byte[] {-128}).hashCode()).isEqualTo(-97); + // Multi-byte: h_0=1, h_1=32, h_2=994, h_3=30817. + assertThat(Binary.fromConstantByteArray(new byte[] {1, 2, 3}).hashCode()) + .isEqualTo(30817); + // "hello" UTF-8 = {104,101,108,108,111}; h_0=1 -> 135 -> 4286 -> 132974 -> 4122302 -> 127791473. + assertThat(Binary.fromString("hello").hashCode()).isEqualTo(127791473); + } + + @Test + public void testHashCodeConsistentAcrossShapes() { + // Contract: a.equals(b) => a.hashCode() == b.hashCode(), regardless of backing shape. + byte[] data = "the-quick-brown-fox-jumps-over-the-lazy-dog".getBytes(StandardCharsets.UTF_8); + byte[] padded = padded(data); + ByteBuffer direct = ByteBuffer.allocateDirect(data.length); + direct.put(data).flip(); + + int h = Binary.fromConstantByteArray(data).hashCode(); + assertThat(Binary.fromConstantByteArray(padded, 5, data.length).hashCode()) + .isEqualTo(h); + assertThat(Binary.fromConstantByteBuffer(ByteBuffer.wrap(data)).hashCode()) + .isEqualTo(h); + assertThat(Binary.fromConstantByteBuffer(direct).hashCode()).isEqualTo(h); + assertThat(Binary.fromString(new String(data, StandardCharsets.UTF_8)).hashCode()) + .isEqualTo(h); + } + + @Test + public void testEqualsAndCompareToMatchScalarOracleAcrossShapesAndLengths() { + Random rnd = new Random(0xBEEFCAFEL); + int[] lengths = {0, 1, 8, 15, 16, 17, 63, 64, 65, 511, 512, 1023}; + for (int len : lengths) { + byte[] a = new byte[len]; + byte[] b = new byte[len]; + rnd.nextBytes(a); + rnd.nextBytes(b); + + Binary aBytes = Binary.fromConstantByteArray(a); + Binary aBuf = Binary.fromConstantByteBuffer(ByteBuffer.wrap(a.clone())); + Binary bBytes = Binary.fromConstantByteArray(b); + Binary bBuf = Binary.fromConstantByteBuffer(ByteBuffer.wrap(b.clone())); + + // equals: byte-identical inputs must be equal across every shape combination. + assertThat(aBytes.equals(aBuf)) + .as("equals byte[]/heapBB len=%d", len) + .isTrue(); + assertThat(aBuf.equals(aBytes)) + .as("equals heapBB/byte[] len=%d", len) + .isTrue(); + + // compareTo: sign must match the independent unsigned oracle. + int expected = scalarCompareUnsigned(a, 0, len, b, 0, len); + assertThat(Integer.signum(aBytes.compareTo(bBytes))) + .as("compareTo byte[]/byte[] len=%d", len) + .isEqualTo(Integer.signum(expected)); + assertThat(Integer.signum(aBytes.compareTo(bBuf))) + .as("compareTo byte[]/heapBB len=%d", len) + .isEqualTo(Integer.signum(expected)); + assertThat(Integer.signum(aBuf.compareTo(bBuf))) + .as("compareTo heapBB/heapBB len=%d", len) + .isEqualTo(Integer.signum(expected)); + + if (len > 0) { + // Flip the middle byte: equals must go false, compareTo must be non-zero. + byte[] aMid = a.clone(); + aMid[len / 2] = (byte) (aMid[len / 2] ^ 0xff); + Binary aMidBin = Binary.fromConstantByteArray(aMid); + assertThat(aBytes.equals(aMidBin)) + .as("mid-mismatch len=%d", len) + .isFalse(); + assertThat(aBytes.compareTo(aMidBin)) + .as("mid-mismatch cmp len=%d", len) + .isNotZero(); + } + } + } + + @Test + public void testCompareToShorterRunsFirstOnPrefixMatch() { + // Arrays.compareUnsigned contract: on prefix match, the shorter run compares less. + Binary shortP = Binary.fromConstantByteArray(new byte[] {1, 2, 3}); + Binary longP = Binary.fromConstantByteArray(new byte[] {1, 2, 3, 4}); + assertThat(shortP.compareTo(longP)).isLessThan(0); + assertThat(longP.compareTo(shortP)).isGreaterThan(0); + assertThat(shortP.compareTo(shortP)).isZero(); + // Unsigned semantics: byte 0xFF (== -1 signed) must compare greater than 0x01. + Binary hi = Binary.fromConstantByteArray(new byte[] {(byte) 0xff}); + Binary lo = Binary.fromConstantByteArray(new byte[] {(byte) 0x01}); + assertThat(hi.compareTo(lo)).isGreaterThan(0); + assertThat(lo.compareTo(hi)).isLessThan(0); + } + + private static int scalarHashCode(byte[] a, int off, int len) { + int h = 1; + for (int i = off; i < off + len; i++) { + h = 31 * h + a[i]; + } + return h; + } + + private static int scalarCompareUnsigned(byte[] a, int aOff, int aLen, byte[] b, int bOff, int bLen) { + int min = Math.min(aLen, bLen); + for (int i = 0; i < min; i++) { + int diff = (a[aOff + i] & 0xff) - (b[bOff + i] & 0xff); + if (diff != 0) return diff; + } + return aLen - bLen; + } }