diff --git a/iaa.cpp b/iaa.cpp index 3acb2c4..f45c3f6 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -197,7 +197,8 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, uint32_t* output_length, qpl_path_t execution_path, - int window_bits, bool* end_of_stream, bool detect_gzip_ext) { + int window_bits, bool* end_of_stream, bool detect_gzip_ext, + bool* window_too_large) { Log(LogLevel::LOG_INFO, "UncompressIAA() Line ", __LINE__, " input_length ", *input_length, "\n"); @@ -235,6 +236,14 @@ int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, qpl_status status = qpl_execute_job(job); if (status != QPL_STS_OK && status != QPL_STS_MORE_OUTPUT_NEEDED) { + // QPL_STS_BAD_DIST_ERR means the stream referenced a match further back + // than IAA's 4 kB history buffer. Unlike the other failures this one is not + // about this call: it says the producer used a larger window, and every + // remaining block of the same stream will be rejected for the same reason. + // Report it separately so the caller can stop submitting. + if (status == QPL_STS_BAD_DIST_ERR && window_too_large != nullptr) { + *window_too_large = true; + } Log(LogLevel::LOG_ERROR, "UncompressIAA() Line ", __LINE__, " qpl_execute_job status ", status, "\n"); return 1; @@ -276,7 +285,7 @@ bool IsIAADecompressible(uint8_t* input, uint32_t input_length, CompressedFormat format = GetCompressedFormat(window_bits); if (format == CompressedFormat::ZLIB) { int window = GetWindowSizeFromZlibHeader(input, input_length); - return window <= 12; + return window <= IAA_MAX_HISTORY_WINDOW_BITS; } // For raw deflate and gzip formats, QPL always reports total_in == // available_in regardless of where BFINAL=1 falls in the stream. This is @@ -298,4 +307,19 @@ bool IsIAADecompressible(uint8_t* input, uint32_t input_length, return input_length > kZipInputStreamBufferSize; } +bool DeclaresIAACompatibleWindow(int window_bits) { + switch (GetCompressedFormat(window_bits)) { + case CompressedFormat::DEFLATE_RAW: + return -window_bits <= IAA_MAX_HISTORY_WINDOW_BITS; + case CompressedFormat::ZLIB: + return window_bits <= IAA_MAX_HISTORY_WINDOW_BITS; + case CompressedFormat::GZIP: + // 16 is the offset zlib adds to select the gzip wrapper; what is left is + // the window. + return window_bits - 16 <= IAA_MAX_HISTORY_WINDOW_BITS; + default: + return false; + } +} + #endif // USE_IAA diff --git a/iaa.h b/iaa.h index 94e1b5b..ff3ac5b 100644 --- a/iaa.h +++ b/iaa.h @@ -16,6 +16,10 @@ inline constexpr unsigned int PREPENDED_BLOCK_LENGTH = 5; inline constexpr unsigned int MAX_BUFFER_SIZE = (2 << 20); +// IAA's decompressor has a fixed 4 kB history buffer, so it can only follow a +// stream whose match distances stay inside a 2^12-byte window. +inline constexpr int IAA_MAX_HISTORY_WINDOW_BITS = 12; + class IAAJob { public: IAAJob() : jobs_(3) {} @@ -55,10 +59,18 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, int window_bits, uint32_t max_compressed_size = 0, bool gzip_ext = false); -int UncompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, - uint32_t* output_length, qpl_path_t execution_path, - int window_bits, bool* end_of_stream, - bool detect_gzip_ext = false); +// window_too_large, when non-null, is set to true if the job was rejected +// because the stream references match distances beyond IAA's fixed 4 kB history +// buffer (QPL_STS_BAD_DIST_ERR). That is a property of whichever compressor +// produced the stream, not of the individual block, so a caller that sees it +// can stop offering the rest of that stream to IAA. It is never set to false; +// the caller owns initialisation. +VISIBLE_FOR_TESTING int UncompressIAA(uint8_t* input, uint32_t* input_length, + uint8_t* output, uint32_t* output_length, + qpl_path_t execution_path, + int window_bits, bool* end_of_stream, + bool detect_gzip_ext = false, + bool* window_too_large = nullptr); VISIBLE_FOR_TESTING bool SupportedOptionsIAA(int window_bits, uint32_t input_length, @@ -68,4 +80,11 @@ VISIBLE_FOR_TESTING bool IsIAADecompressible(uint8_t* input, uint32_t input_length, int window_bits); +// True if window_bits declares a maximum window IAA's history buffer can +// follow, whatever the format's own header says. inflateReset2() takes such a +// declaration from the caller, and it is a stronger statement than a remembered +// rejection: a stream that referenced further back than this would be refused +// by zlib too. +VISIBLE_FOR_TESTING bool DeclaresIAACompatibleWindow(int window_bits); + #endif // USE_IAA diff --git a/tests/inflate_test.cpp b/tests/inflate_test.cpp index 2819441..831044c 100644 --- a/tests/inflate_test.cpp +++ b/tests/inflate_test.cpp @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // inflate() regression suites: the IGZIP inflate path, the accelerator -> -// IGZIP fallbacks, the flush/data_type gate and the dictionary fallback. +// IGZIP fallbacks, the flush/data_type gate, the dictionary fallback and the +// remembered IAA history-window rejection. #include @@ -11,6 +12,7 @@ #include #include "../config/config.h" +#include "../iaa.h" #include "../zlib_accel.h" #include "test_utils.h" @@ -2049,3 +2051,613 @@ TEST_F(DictionaryMidstreamFallbackRegressionTest, } #endif #endif + +#ifdef USE_IAA +// IAA's decompressor has a fixed 4 kB history buffer, so it cannot decode a +// stream whose producer used a larger window -- zlib's default is 32 kB. QPL +// reports that as QPL_STS_BAD_DIST_ERR, and it is the one decompress failure +// that predicts the next call: the window belongs to the compressor, not to the +// block. IsIAADecompressible() cannot see it for raw deflate or gzip, where +// there is no header to read the window out of, so the only way to know is to +// be told once and remember. +class IAAWindowRejectionTest : public ::testing::Test { + protected: + void SetUp() override { + saved_use_zlib_compress_ = GetConfig(USE_ZLIB_COMPRESS); + saved_use_iaa_compress_ = GetConfig(USE_IAA_COMPRESS); + saved_use_qat_compress_ = GetConfig(USE_QAT_COMPRESS); + saved_use_igzip_compress_ = GetConfig(USE_IGZIP_COMPRESS); + saved_use_zlib_uncompress_ = GetConfig(USE_ZLIB_UNCOMPRESS); + saved_use_iaa_uncompress_ = GetConfig(USE_IAA_UNCOMPRESS); + saved_use_qat_uncompress_ = GetConfig(USE_QAT_UNCOMPRESS); + saved_use_igzip_uncompress_ = GetConfig(USE_IGZIP_UNCOMPRESS); + // SetCompressPath/SetUncompressPath write these two unconditionally, and + // the no-fallback case below turns the third off, so all three have to come + // back or this fixture makes the suite order-dependent. + saved_iaa_prepend_empty_block_ = GetConfig(IAA_PREPEND_EMPTY_BLOCK); + saved_qat_allow_chunking_ = GetConfig(QAT_COMPRESSION_ALLOW_CHUNKING); + saved_igzip_fallback_ = GetConfig(IGZIP_FALLBACK); + saved_iaa_uncompress_percentage_ = GetConfig(IAA_UNCOMPRESS_PERCENTAGE); + } + + void TearDown() override { + SetConfig(USE_ZLIB_COMPRESS, saved_use_zlib_compress_); + SetConfig(USE_IAA_COMPRESS, saved_use_iaa_compress_); + SetConfig(USE_QAT_COMPRESS, saved_use_qat_compress_); + SetConfig(USE_IGZIP_COMPRESS, saved_use_igzip_compress_); + SetConfig(USE_ZLIB_UNCOMPRESS, saved_use_zlib_uncompress_); + SetConfig(USE_IAA_UNCOMPRESS, saved_use_iaa_uncompress_); + SetConfig(USE_QAT_UNCOMPRESS, saved_use_qat_uncompress_); + SetConfig(USE_IGZIP_UNCOMPRESS, saved_use_igzip_uncompress_); + SetConfig(IAA_PREPEND_EMPTY_BLOCK, saved_iaa_prepend_empty_block_); + SetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, saved_qat_allow_chunking_); + SetConfig(IGZIP_FALLBACK, saved_igzip_fallback_); + SetConfig(IAA_UNCOMPRESS_PERCENTAGE, saved_iaa_uncompress_percentage_); + } + + private: + uint32_t saved_use_zlib_compress_ = 0; + uint32_t saved_use_iaa_compress_ = 0; + uint32_t saved_use_qat_compress_ = 0; + uint32_t saved_use_igzip_compress_ = 0; + uint32_t saved_use_zlib_uncompress_ = 0; + uint32_t saved_use_iaa_uncompress_ = 0; + uint32_t saved_use_qat_uncompress_ = 0; + uint32_t saved_use_igzip_uncompress_ = 0; + uint32_t saved_iaa_prepend_empty_block_ = 0; + uint32_t saved_qat_allow_chunking_ = 0; + uint32_t saved_igzip_fallback_ = 0; + uint32_t saved_iaa_uncompress_percentage_ = 0; +}; + +// Run one whole stream through strm and check the bytes. Returns the last +// inflate() code so the caller can assert on it, or Z_DATA_ERROR if the output +// came back wrong. +static int InflateWholeStream(z_streamp strm, const std::string& compressed, + const char* expected, size_t expected_length) { + // A whole-stream decode finishes or errors out in one or two calls; this is + // headroom, not an expected count. Exhausting it below is reported rather + // than left to surface as a plain return-code mismatch, since "stuck" and + // "wrong answer" want different diagnostics. + constexpr int kMaxInflateCalls = 128; + std::vector output(expected_length + 1024); + strm->next_in = + reinterpret_cast(const_cast(compressed.data())); + strm->avail_in = static_cast(compressed.size()); + strm->next_out = output.data(); + strm->avail_out = static_cast(output.size()); + int ret = Z_OK; + int guard = 0; + for (; guard < kMaxInflateCalls; guard++) { + ret = inflate(strm, Z_NO_FLUSH); + if (ret != Z_OK && ret != Z_BUF_ERROR) { + break; + } + } + if (guard == kMaxInflateCalls && (ret == Z_OK || ret == Z_BUF_ERROR)) { + ADD_FAILURE() << "inflate() neither finished nor errored after " + << kMaxInflateCalls << " calls; decoder appears stuck"; + } + if (ret != Z_STREAM_END) { + return ret; + } + if (strm->total_out != expected_length || + memcmp(output.data(), expected, expected_length) != 0) { + return Z_DATA_ERROR; + } + return Z_STREAM_END; +} + +// Every case below except the first needs QPL to get far enough into a job to +// report the oversized window. Without a usable device it fails at job +// initialization instead, which leaves the flag correctly clear -- so the test +// would be asserting the opposite of what it means to. Probe with a stream IAA +// can definitely decode: a short one, whose matches cannot reach back 4 kB +// because the whole payload is smaller than that. +static bool IAAHardwareDecompressWorks() { + const size_t input_length = 2048; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x4144); + if (input == nullptr) { + return false; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + std::string compressed; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + DestroyBlock(input); + if (ret != Z_STREAM_END) { + return false; + } + + std::vector output(input_length + 1024); + uint32_t input_len = static_cast(compressed.size()); + uint32_t output_len = static_cast(output.size()); + bool end_of_stream = false; + ret = UncompressIAA(reinterpret_cast(&compressed[0]), &input_len, + output.data(), &output_len, qpl_path_hardware, + /*window_bits=*/-15, &end_of_stream); + return ret == 0 && end_of_stream && output_len == input_length; +} + +// The contract UncompressIAA() now offers its callers, checked on QPL's +// software path so that it holds on a host with no device: the software path +// rejects an oversized window for the same reason and with the same status. +TEST_F(IAAWindowRejectionTest, UncompressIAAReportsOversizedWindow) { + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e11); + ASSERT_NE(input, nullptr); + + // Two encodings of the same bytes. GenerateSeededCompressibleBlock() repeats + // a string every 8192 bytes, so with zlib's full window the first is + // guaranteed to contain a match distance IAA cannot reach; restricted to 4 + // kB, the second cannot contain one. + std::string wide; + std::string narrow; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &wide, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + ASSERT_EQ(ZlibCompress(input, input_length, &narrow, -12, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + std::vector output(input_length + 1024); + uint32_t input_len = static_cast(wide.size()); + uint32_t output_len = static_cast(output.size()); + bool end_of_stream = false; + bool window_too_large = false; + EXPECT_NE(UncompressIAA(reinterpret_cast(&wide[0]), &input_len, + output.data(), &output_len, qpl_path_software, + /*window_bits=*/-15, &end_of_stream, + /*detect_gzip_ext=*/false, &window_too_large), + 0); + EXPECT_TRUE(window_too_large); + + // A stream IAA can follow decodes, and leaves the flag alone. Never setting + // it to false is what lets a caller pass one bool through a whole stream. + input_len = static_cast(narrow.size()); + output_len = static_cast(output.size()); + end_of_stream = false; + bool narrow_window_too_large = false; + EXPECT_EQ(UncompressIAA(reinterpret_cast(&narrow[0]), &input_len, + output.data(), &output_len, qpl_path_software, + /*window_bits=*/-12, &end_of_stream, + /*detect_gzip_ext=*/false, &narrow_window_too_large), + 0); + EXPECT_FALSE(narrow_window_too_large); + EXPECT_TRUE(end_of_stream); + EXPECT_EQ(output_len, input_length); + EXPECT_EQ(memcmp(output.data(), input, input_length), 0); + + // The case the block above cannot tell apart from a no-op: starting from an + // already-true flag, so that an implementation which clears the + // out-parameter on every successful call -- rather than leaving an untouched + // one alone -- would be caught rather than passing vacuously. + input_len = static_cast(narrow.size()); + output_len = static_cast(output.size()); + end_of_stream = false; + bool already_true = true; + EXPECT_EQ(UncompressIAA(reinterpret_cast(&narrow[0]), &input_len, + output.data(), &output_len, qpl_path_software, + /*window_bits=*/-12, &end_of_stream, + /*detect_gzip_ext=*/false, &already_true), + 0); + EXPECT_TRUE(already_true); + + // Omitting the out-parameter has to stay legal: most callers do not care + // which failure they got. + input_len = static_cast(wide.size()); + output_len = static_cast(output.size()); + end_of_stream = false; + EXPECT_NE(UncompressIAA(reinterpret_cast(&wide[0]), &input_len, + output.data(), &output_len, qpl_path_software, + /*window_bits=*/-15, &end_of_stream), + 0); + + DestroyBlock(input); +} + +// The point of the whole change. A rejection has to outlive inflateReset(), +// because a reset is exactly what the callers that matter do between documents: +// Lucene resets its Inflater once per stored field. Clearing the flag on reset +// would forget the lesson before it was ever acted on. +TEST_F(IAAWindowRejectionTest, StreamFlagSurvivesInflateReset) { + if (!IAAHardwareDecompressWorks()) { + GTEST_SKIP() << "no usable IAA device: QPL cannot reach the point where it " + "reports an oversized history window"; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e12); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + EXPECT_FALSE(InflateIAAWindowRejected(&stream)); + + // The rejection costs one submission and then falls through to zlib, so the + // bytes are still right. + EXPECT_EQ(InflateWholeStream(&stream, compressed, input, input_length), + Z_STREAM_END); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + + ASSERT_EQ(inflateReset(&stream), Z_OK); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + // inflateReset() clears the path, so a stream that had forgotten the + // rejection would be dispatched to IAA again here. + EXPECT_EQ(GetInflateExecutionPath(&stream), UNDEFINED); + EXPECT_EQ(InflateWholeStream(&stream, compressed, input, input_length), + Z_STREAM_END); + EXPECT_NE(GetInflateExecutionPath(&stream), IAA); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + + // inflateReset2() restarts the stream in a new format, and is the other way + // back to an undefined path. + ASSERT_EQ(inflateReset2(&stream, -15), Z_OK); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + + ASSERT_EQ(inflateEnd(&stream), Z_OK); + DestroyBlock(input); +} + +// A copy decodes the rest of the same stream, so it inherits the verdict. The +// settings are rebuilt member by member in SetFromCopy(), not assigned, so this +// is the kind of field that gets silently dropped. +TEST_F(IAAWindowRejectionTest, StreamFlagPropagatesThroughInflateCopy) { + if (!IAAHardwareDecompressWorks()) { + GTEST_SKIP() << "no usable IAA device: QPL cannot reach the point where it " + "reports an oversized history window"; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e13); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + z_stream source; + memset(&source, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&source, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&source, compressed, input, input_length), + Z_STREAM_END); + ASSERT_TRUE(InflateIAAWindowRejected(&source)); + + z_stream dest; + memset(&dest, 0, sizeof(z_stream)); + ASSERT_EQ(inflateCopy(&dest, &source), Z_OK); + EXPECT_TRUE(InflateIAAWindowRejected(&dest)); + // And the copy keeps it across its own reset, like the original. + ASSERT_EQ(inflateReset(&dest), Z_OK); + EXPECT_TRUE(InflateIAAWindowRejected(&dest)); + EXPECT_EQ(InflateWholeStream(&dest, compressed, input, input_length), + Z_STREAM_END); + EXPECT_NE(GetInflateExecutionPath(&dest), IAA); + + ASSERT_EQ(inflateEnd(&dest), Z_OK); + ASSERT_EQ(inflateEnd(&source), Z_OK); + DestroyBlock(input); +} + +// A stream IAA can serve must not be tarred by another stream's rejection: the +// flag is per stream, and there is no process-wide counter behind it. +TEST_F(IAAWindowRejectionTest, RejectionDoesNotAffectOtherStreams) { + if (!IAAHardwareDecompressWorks()) { + GTEST_SKIP() << "no usable IAA device: QPL cannot reach the point where it " + "reports an oversized history window"; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e14); + ASSERT_NE(input, nullptr); + + std::string wide; + std::string narrow; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &wide, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + ASSERT_EQ(ZlibCompress(input, input_length, &narrow, -12, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + z_stream rejected; + memset(&rejected, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&rejected, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&rejected, wide, input, input_length), + Z_STREAM_END); + ASSERT_TRUE(InflateIAAWindowRejected(&rejected)); + + z_stream served; + memset(&served, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&served, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&served, narrow, input, input_length), + Z_STREAM_END); + EXPECT_FALSE(InflateIAAWindowRejected(&served)); + EXPECT_EQ(GetInflateExecutionPath(&served), IAA); + + ASSERT_EQ(inflateEnd(&served), Z_OK); + ASSERT_EQ(inflateEnd(&rejected), Z_OK); + DestroyBlock(input); +} + +// The window a caller declares, independent of any hardware: the format's +// wrapper offset has to come off before the window is compared, or a gzip +// stream looks like a 24-bit window and a raw one like a negative window. +TEST_F(IAAWindowRejectionTest, DeclaresIAACompatibleWindowFollowsTheFormat) { + // Raw deflate. + EXPECT_TRUE(DeclaresIAACompatibleWindow(-8)); + EXPECT_TRUE(DeclaresIAACompatibleWindow(-12)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(-13)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(-15)); + // Zlib. + EXPECT_TRUE(DeclaresIAACompatibleWindow(8)); + EXPECT_TRUE(DeclaresIAACompatibleWindow(12)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(13)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(15)); + // Gzip, which zlib selects by adding 16. + EXPECT_TRUE(DeclaresIAACompatibleWindow(16 + 8)); + EXPECT_TRUE(DeclaresIAACompatibleWindow(16 + 12)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(16 + 13)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(16 + 15)); + // Anything this shim does not map to a format cannot be declared compatible, + // including zlib's automatic-detection range, where the stream picks the + // wrapper and the caller has therefore declared nothing. + EXPECT_FALSE(DeclaresIAACompatibleWindow(0)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(32 + 15)); + EXPECT_FALSE(DeclaresIAACompatibleWindow(-16)); +} + +// A narrowing inflateReset2() is the caller declaring the next stream's window, +// so it retires the verdict: the flag is an inference about the previous +// stream's producer, and a declaration outranks an inference. Without this a +// stream that was rejected once never reaches IAA again even after the caller +// has said the data cannot reference beyond 4 kB. +TEST_F(IAAWindowRejectionTest, NarrowingInflateReset2ClearsTheVerdict) { + if (!IAAHardwareDecompressWorks()) { + GTEST_SKIP() << "no usable IAA device: QPL cannot reach the point where it " + "reports an oversized history window"; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e15); + ASSERT_NE(input, nullptr); + + std::string wide; + std::string narrow; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &wide, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + ASSERT_EQ(ZlibCompress(input, input_length, &narrow, -12, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&stream, wide, input, input_length), + Z_STREAM_END); + ASSERT_TRUE(InflateIAAWindowRejected(&stream)); + + // Same window, so nothing has been declared and the verdict stands. This is + // the control: without it the test would pass on a build that cleared the + // flag on every reset. + ASSERT_EQ(inflateReset2(&stream, -15), Z_OK); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + EXPECT_EQ(InflateWholeStream(&stream, narrow, input, input_length), + Z_STREAM_END); + EXPECT_NE(GetInflateExecutionPath(&stream), IAA); + + // Narrowing to a window IAA can follow retires it, and the next stream is + // offered to IAA again -- and served, since the bytes really do stay inside + // 4 kB. + ASSERT_EQ(inflateReset2(&stream, -12), Z_OK); + EXPECT_FALSE(InflateIAAWindowRejected(&stream)); + EXPECT_EQ(InflateWholeStream(&stream, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IAA); + + ASSERT_EQ(inflateEnd(&stream), Z_OK); + DestroyBlock(input); +} + +// The remembered verdict is an optimization, so it must not change what a +// caller who has turned every fallback off gets back. That configuration +// answers Z_DATA_ERROR whenever no engine will take the data -- an input below +// IAA's 512-byte floor does it with no verdict involved -- but the verdict must +// not be what puts a stream in that category: with nothing else to fall back +// to, the stream is offered to IAA anyway, because a job that probably fails +// beats refusing data that may decode. +TEST_F(IAAWindowRejectionTest, RememberedRejectionWithNoFallbackDoesNotRefuse) { + if (!IAAHardwareDecompressWorks()) { + GTEST_SKIP() << "no usable IAA device: QPL cannot reach the point where it " + "reports an oversized history window"; + } + SetCompressPath(ZLIB, /*zlib_fallback=*/true, false, false); + + const size_t input_length = 64 * 1024; + char* input = GenerateSeededCompressibleBlock(input_length, /*seed=*/0x7e16); + ASSERT_NE(input, nullptr); + const size_t tiny_length = 256; + char* tiny = GenerateSeededCompressibleBlock(tiny_length, /*seed=*/0x7e17); + ASSERT_NE(tiny, nullptr); + + std::string wide; + std::string narrow; + std::string tiny_compressed; + size_t output_upper_bound = 0; + ExecutionPath compress_path = UNDEFINED; + ASSERT_EQ(ZlibCompress(input, input_length, &wide, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + ASSERT_EQ(ZlibCompress(input, input_length, &narrow, -12, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + ASSERT_EQ(ZlibCompress(tiny, tiny_length, &tiny_compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path), + Z_STREAM_END); + + // IAA and nothing else: no zlib, no igzip retry. + SetUncompressPath(IAA, /*zlib_fallback=*/false, false); + SetConfig(IGZIP_FALLBACK, 0); + + // The floor case, which no part of this change touches: a stream too short + // for IsIAADecompressible() reaches no engine and is refused. This is the + // configuration's own contract, and it stays exactly as it was. + z_stream small; + memset(&small, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&small, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&small, tiny_compressed, tiny, tiny_length), + Z_DATA_ERROR); + EXPECT_FALSE(InflateIAAWindowRejected(&small)); + ASSERT_EQ(inflateEnd(&small), Z_OK); + + // A stream IAA can follow is still served, so the configuration is not + // simply broken. + z_stream served; + memset(&served, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&served, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&served, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&served), IAA); + ASSERT_EQ(inflateEnd(&served), Z_OK); + + // The rejection itself is refused with no fallback to hand it to, which is + // what this configuration means and is already true without the verdict. + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&stream, wide, input, input_length), + Z_DATA_ERROR); + ASSERT_TRUE(InflateIAAWindowRejected(&stream)); + + // The next stream on the same z_stream is a payload IAA can follow, and the + // verdict does not get to refuse it: with no other engine available the + // suppression stands down and IAA is submitted, so the caller gets the same + // answer as `served` above. The verdict is still recorded -- it just is not + // deciding anything here. + ASSERT_EQ(inflateReset(&stream), Z_OK); + EXPECT_EQ(InflateWholeStream(&stream, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IAA); + EXPECT_TRUE(InflateIAAWindowRejected(&stream)); + + // Narrowing the window retires the verdict outright, with no fallback too. + ASSERT_EQ(inflateReset2(&stream, -12), Z_OK); + EXPECT_FALSE(InflateIAAWindowRejected(&stream)); + EXPECT_EQ(InflateWholeStream(&stream, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IAA); + + ASSERT_EQ(inflateEnd(&stream), Z_OK); + +#ifdef USE_IGZIP + // With zlib off but IGZIP on, the verdict does its job again: another engine + // can take the stream, so IAA is suppressed and IGZIP serves it. Without the + // suppression the doomed IAA submission would go first and, with no fallback + // behind it, refuse data IGZIP would have decoded. + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + + z_stream to_igzip; + memset(&to_igzip, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&to_igzip, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_igzip, wide, input, input_length), + Z_DATA_ERROR); + ASSERT_TRUE(InflateIAAWindowRejected(&to_igzip)); + + ASSERT_EQ(inflateReset(&to_igzip), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_igzip, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&to_igzip), IGZIP); + ASSERT_EQ(inflateEnd(&to_igzip), Z_OK); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); +#endif + +#ifdef USE_QAT + // Unlike IGZIP above, enabling QAT here does not stand the suppression + // down: QAT's eligibility is a buffer/window check with no view of whether + // a device is present, so treating it as a safe alternative could turn a + // stream IAA can decode into a refusal on a host where QAT is compiled in + // but not functional -- the reason inflate() does not count qat_available + // among the terms that let it suppress IAA. With QAT the only other engine + // enabled, the verdict therefore stays out of the decision and IAA gets + // the same chance at the narrow stream as it would with no other engine at + // all, which is exactly what the traffic split pinned at 100% IAA is here + // to make unambiguous. + SetConfig(USE_QAT_UNCOMPRESS, 1); + SetConfig(IAA_UNCOMPRESS_PERCENTAGE, 100); + + z_stream to_qat; + memset(&to_qat, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&to_qat, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_qat, wide, input, input_length), + Z_DATA_ERROR); + ASSERT_TRUE(InflateIAAWindowRejected(&to_qat)); + + ASSERT_EQ(inflateReset(&to_qat), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_qat, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&to_qat), IAA); + ASSERT_EQ(inflateEnd(&to_qat), Z_OK); +#endif + +#if defined(USE_IGZIP) && defined(USE_QAT) + // The mixed configuration the suppression's igzip_available term still got + // wrong even after the qat_available fix above: with QAT also eligible, + // the selection ladder picks QAT ahead of IGZIP regardless of which one + // justified suppressing IAA, and the accelerator->IGZIP retry only covers + // that QAT choice when IGZIP_FALLBACK is set -- left off here, as in every + // other block in this test. So igzip_available alone must not stand the + // suppression down while qat_available is also true; the traffic split, + // still pinned at 100% IAA from the block above, is what actually serves + // the narrow stream. + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + + z_stream to_mixed; + memset(&to_mixed, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&to_mixed, -15), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_mixed, wide, input, input_length), + Z_DATA_ERROR); + ASSERT_TRUE(InflateIAAWindowRejected(&to_mixed)); + + ASSERT_EQ(inflateReset(&to_mixed), Z_OK); + EXPECT_EQ(InflateWholeStream(&to_mixed, narrow, input, input_length), + Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&to_mixed), IAA); + ASSERT_EQ(inflateEnd(&to_mixed), Z_OK); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); +#endif + + DestroyBlock(tiny); + DestroyBlock(input); +} + +#endif // USE_IAA diff --git a/zlib_accel.cpp b/zlib_accel.cpp index d4a9879..e299365 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -428,6 +428,14 @@ struct InflateSettings { // clears the state; ISA-L does not, and keeps parsing whatever follows the // rejected bytes as a new block header, so the latch has to live here. bool data_error = false; + // Set once IAA has rejected a block of this stream for referencing a match + // beyond its 4 kB history buffer. Deliberately NOT cleared by inflateReset: + // the window is a property of the compressor that produced the bytes, and a + // reset starts a new stream from the same producer in every caller that + // matters here (Lucene resets its Inflater once per stored-field document). + // Clearing it would make the flag useless, since almost every rejection + // arrives on a stream that is about to be reset. + bool iaa_window_too_large = false; }; // isal_strm is a raw pointer, so destroying a settings object does not free the @@ -545,6 +553,12 @@ class InflateStreamSettings { settings->stream_end_reached = source.stream_end_reached; settings->bytes_consumed = source.bytes_consumed; settings->data_error = source.data_error; + // A copy decodes the rest of the same stream, so it inherits what IAA + // already said about that stream's history window. This has to be copied + // out by hand like every other field: the settings are rebuilt member by + // member here, not assigned, so a new member is silently dropped + // otherwise. + settings->iaa_window_too_large = source.iaa_window_too_large; map.Set(dest, std::move(settings)); } catch (...) { Log(LogLevel::LOG_ERROR, @@ -625,6 +639,16 @@ static void ResetDeflateStreamState( // Same for the inflate side. A reset stream is ready to decode again; leaving // the terminal state set would wedge every later inflate() at Z_STREAM_END. +// +// iaa_window_too_large deliberately does NOT belong here. It records what IAA +// said about the compressor that produced these bytes, and a reset stream is +// almost always the same caller decoding more output from the same producer -- +// Lucene resets its Inflater once per stored-field document. Clearing it here +// would make the flag useless: it would be forgotten before it was ever +// consulted, and the shim would go back to submitting jobs it knows will be +// rejected. inflateReset2() is the one exception, and clears the field itself: +// a caller that declares an IAA-sized window has said what the next stream is, +// which beats an inference drawn from the last one. static void ResetInflateStreamState( const std::shared_ptr& settings) { if (settings == nullptr) { @@ -1756,6 +1780,35 @@ int ZEXPORT inflate(z_streamp strm, int flush) { igzip_available = igzip_supported_options; #endif +#ifdef USE_IAA + // IsIAADecompressible cannot see match distances, so for raw deflate and + // gzip it has no header to read and is guessing. iaa_window_too_large is + // what a wrong guess, once made, costs being remembered: IAA has already + // told us this stream's producer used a window it cannot follow. + // + // That memory is an optimization, and an optimization must not change the + // answer, so it only suppresses IAA while some other engine can take the + // stream -- and "can take" has to mean will actually run it, not merely + // pass an eligibility check. zlib is always that safe: the fall-through + // below is software with no device to be missing. IGZIP is software too, + // but eligible is not enough on its own: the selection ladder below picks + // QAT ahead of IGZIP whenever qat_available is also true, and QAT's + // eligibility (qat_available) is a buffer/window check with no view of + // whether a device exists -- the reason it is left out of this condition + // entirely. So a stream this suppression hands to "IGZIP" can really be + // handed to a QAT that then fails, and the accelerator retry below only + // reaches IGZIP when IGZIP_FALLBACK is also set; igzip_available alone + // promises nothing about which engine the ladder actually picks. With no + // term true a suppressed stream would be refused outright, so submit it + // and let IAA decide: a job that probably fails beats refusing data that + // may decode. + if (iaa_available && inflate_settings->iaa_window_too_large && + (configs[USE_ZLIB_UNCOMPRESS] || + (igzip_available && (!qat_available || configs[IGZIP_FALLBACK])))) { + iaa_available = false; + } +#endif + // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; @@ -1778,9 +1831,10 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (path_selected == IAA) { #ifdef USE_IAA in_call = true; - ret = UncompressIAA(strm->next_in, &input_len, strm->next_out, - &output_len, qpl_path_hardware, - inflate_settings->window_bits, &end_of_stream); + ret = UncompressIAA( + strm->next_in, &input_len, strm->next_out, &output_len, + qpl_path_hardware, inflate_settings->window_bits, &end_of_stream, + /*detect_gzip_ext=*/false, &inflate_settings->iaa_window_too_large); SetInflatePath(inflate_settings, IAA); // IAA inflate is stateless in this wrapper. If stream end was not // reached, use zlib for stateful continuation. @@ -2088,6 +2142,19 @@ int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { ResetInflateStreamState(inflate_settings); inflate_settings->window_bits = windowBits; +#ifdef USE_IAA + // The one thing that overrides a remembered rejection. That verdict is an + // inference about the compressor that produced the previous stream, and + // inflateReset2() is the caller stating outright what the next stream's + // window is; a declaration IAA can follow wins over the inference, since + // bytes that reached further back would be refused by zlib as well. An + // inflateReset() carries no such statement, which is why the verdict + // survives it. + if (DeclaresIAACompatibleWindow(windowBits)) { + inflate_settings->iaa_window_too_large = false; + } +#endif + if (inflate_settings->isal_strm != nullptr) { #ifdef USE_IGZIP // isal_inflate_reset() deliberately preserves crc_flag and hist_bits, and @@ -2429,6 +2496,11 @@ bool InflateOwnsIgzipState(z_streamp strm) { return inflate_settings != nullptr && inflate_settings->isal_strm != nullptr; } +bool InflateIAAWindowRejected(z_streamp strm) { + auto inflate_settings = inflate_stream_settings.Get(strm); + return inflate_settings != nullptr && inflate_settings->iaa_window_too_large; +} + enum class FileMode { NONE, READ, WRITE, APPEND }; // Beside the enum rather than beside its first caller. Every gz entry point @@ -3197,6 +3269,13 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, bool igzip_available = false; #ifdef USE_IAA + // No remembered window rejection here, unlike inflate(): gzread pins a file + // to zlib on its first accelerator failure of any kind (see + // use_zlib_for_decompression at the call site), so nothing more is submitted + // for the rest of that read pass. gzrewind clears the pin deliberately, to + // offer the accelerator another try at a file whose stream a mid-file + // fallback had taken away. One wasted submission per pass is all this path + // can spend, and within a pass there is no second one to suppress. iaa_available = configs[USE_IAA_UNCOMPRESS] && SupportedOptionsIAA(kWindowBitsGzip, *input_length, *output_length) && diff --git a/zlib_accel.h b/zlib_accel.h index ed21080..3df92ee 100644 --- a/zlib_accel.h +++ b/zlib_accel.h @@ -2,6 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once + +// The block below pushes default visibility across the #include, which is +// what gives every ZEXPORT definition in zlib_accel.cpp the visibility +// deflate()/inflate()/gzopen()/etc. need to be interposed at all -- it is not +// a place to add a test accessor. A declaration added there is exported the +// same way the real zlib API is, indistinguishable from it in the built +// .so's symbol table and with no attribute of its own to say otherwise. Use +// this macro after the pop instead, the same convention iaa.h/qat.h/igzip.h +// use for symbols a test binary needs from an otherwise hidden library. +#define VISIBLE_FOR_TESTING __attribute__((visibility("default"))) + #pragma GCC visibility push(default) #include @@ -24,3 +35,9 @@ bool DeflateOwnsIgzipState(z_streamp strm); bool InflateOwnsIgzipState(z_streamp strm); #pragma GCC visibility pop + +// True once IAA has rejected a block of this stream for referencing a match +// beyond its 4 kB history buffer. Tests need it because the record deliberately +// survives inflateReset(), and nothing else about the stream reveals that it is +// being kept. Always false in a build without IAA support. +VISIBLE_FOR_TESTING bool InflateIAAWindowRejected(z_streamp strm);