From afcb1cf37bcd508c99b9195d723e0f0b43c90f40 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 18 Sep 2026 13:03:08 -0700 Subject: [PATCH 1/4] Support StreamReadConstraints.maxDocumentLength and maxTokenCount in Avro parser (#785) - AvroParserImpl.nextToken(): route through _nullSafeUpdateToken() so tokens are counted (nextFieldName() variants already did this) - JacksonAvroParserImpl: validate document length on buffer reload - AvroFactory / ApacheAvroFactory: validate document length up front for byte[] input - ApacheAvroParserImpl: wrap the InputStream in a byte-counting stream (only when maxDocumentLength is set), since the Apache BinaryDecoder does its own buffering Backport of #786 (3.x) to 2.18. Co-Authored-By: PJ Fanning Co-Authored-By: Claude Opus 5 (1M context) --- .../jackson/dataformat/avro/AvroFactory.java | 2 + .../avro/apacheimpl/ApacheAvroFactory.java | 2 + .../avro/apacheimpl/ApacheAvroParserImpl.java | 60 ++++++++++ .../dataformat/avro/deser/AvroParserImpl.java | 3 +- .../avro/deser/JacksonAvroParserImpl.java | 2 + .../avro/dos/LongDocumentAvroReadTest.java | 108 ++++++++++++++++++ .../avro/dos/TokenCountAvroReadTest.java | 90 +++++++++++++++ release-notes/CREDITS-2.x | 3 + release-notes/VERSION-2.x | 6 + 9 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java index 3fd43ffa7..66da00688 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java @@ -439,6 +439,8 @@ protected JsonParser _createParser(char[] data, int offset, int len, IOContext c @Override protected AvroParser _createParser(byte[] data, int offset, int len, IOContext ctxt) throws IOException { + // [core#1548] Validate doc length up front for fixed buffers + _streamReadConstraints.validateDocumentLength(len); // !!! 21-Apr-2017, tatu: make configurable return new JacksonAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, // return new ApacheAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java index a15bc96fd..1f6d54469 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java @@ -51,6 +51,8 @@ protected AvroParser _createParser(InputStream in, IOContext ctxt) throws IOExce @Override protected AvroParser _createParser(byte[] data, int offset, int len, IOContext ctxt) throws IOException { + // [core#1548] Validate doc length up front for fixed buffers + _streamReadConstraints.validateDocumentLength(len); return new ApacheAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, _avroRecyclerPool.acquireAndLinkPooled(), _objectCodec, data, offset, len); diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java index ab59a3979..d54e82876 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java @@ -1,5 +1,6 @@ package com.fasterxml.jackson.dataformat.avro.apacheimpl; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Writer; @@ -93,6 +94,11 @@ public ApacheAvroParserImpl(IOContext ctxt, int parserFeatures, int avroFeatures _bufferRecyclable = true; _apacheCodecRecycler = apacheCodecRecycler; + // [dataformats-binary#785] Apache decoder does its own buffering, so document + // length constraint has to be applied by counting bytes it pulls from the stream + if (_streamReadConstraints.hasMaxDocumentLength()) { + in = new LengthCheckingInputStream(in, _streamReadConstraints); + } final boolean buffering = Feature.AVRO_BUFFERING.enabledIn(avroFeatures); BinaryDecoder decoderToReuse = apacheCodecRecycler.acquireDecoder(); _decoder = buffering @@ -412,4 +418,58 @@ protected JsonToken setString(String str) { _textValue = str; return JsonToken.VALUE_STRING; } + + /* + /********************************************************** + /* Helper classes + /********************************************************** + */ + + /** + * {@link InputStream} wrapper that applies {@link StreamReadConstraints#validateDocumentLength} + * to the number of bytes read so far, for use with Apache {@link BinaryDecoder} which + * reads from the stream directly. + * + * @since 2.18.11 + */ + private final static class LengthCheckingInputStream extends FilterInputStream + { + private final StreamReadConstraints _constraints; + + private long _bytesRead; + + LengthCheckingInputStream(InputStream in, StreamReadConstraints constraints) { + super(in); + _constraints = constraints; + } + + @Override + public int read() throws IOException { + int b = in.read(); + if (b >= 0) { + _constraints.validateDocumentLength(++_bytesRead); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int count = in.read(b, off, len); + if (count > 0) { + _bytesRead += count; + _constraints.validateDocumentLength(_bytesRead); + } + return count; + } + + @Override + public long skip(long n) throws IOException { + long count = in.skip(n); + if (count > 0) { + _bytesRead += count; + _constraints.validateDocumentLength(_bytesRead); + } + return count; + } + } } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java index d54893503..b402a8707 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java @@ -95,8 +95,7 @@ public JsonToken nextToken() throws IOException _enumIndex = -1; _binaryValue = null; JsonToken t = _avroContext.nextToken(); - _currToken = t; - return t; + return _nullSafeUpdateToken(t); } @Override diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java index e3f74313f..0783cc171 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java @@ -1060,6 +1060,7 @@ protected final boolean _loadMore() throws IOException if (_inputStream != null) { int count = _inputStream.read(_inputBuffer, 0, _inputBuffer.length); _currInputProcessed += _inputEnd; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); _inputPtr = 0; if (count > 0) { _inputEnd = count; @@ -1086,6 +1087,7 @@ protected final void _loadToHaveAtLeast(int minAvailable) throws IOException // Need to move remaining data in front? int amount = _inputEnd - _inputPtr; _currInputProcessed += _inputPtr; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); if (_inputPtr > 0) { if (amount > 0) { //_currInputRowStart -= _inputPtr; diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java new file mode 100644 index 000000000..9140dcae9 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java @@ -0,0 +1,108 @@ +package com.fasterxml.jackson.dataformat.avro.dos; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.apacheimpl.ApacheAvroFactory; + +// [dataformats-binary#785]: `StreamReadConstraints.maxDocumentLength` for Avro +public class LongDocumentAvroReadTest extends AvroTestBase +{ + public static class Item { + public String id; + public int size; + public long stuff; + } + + public static class Items { + public List items = new ArrayList<>(); + } + + private final static int MAX_DOC_LEN = 50_000; + + private final AvroMapper MAPPER_VANILLA = newMapper(); + + private final AvroSchema ITEMS_SCHEMA; + { + try { + ITEMS_SCHEMA = MAPPER_VANILLA.schemaFor(Items.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void testLongDocumentConstraint() throws Exception + { + // Need a bit longer than minimum since checking is approximate, not exact + byte[] doc = createBigDoc(60_000); + for (boolean apache : new boolean[] { false, true }) { + AvroMapper mapper = constrainedMapper(apache); + _testLongDocumentConstraint(mapper, doc, true); + _testLongDocumentConstraint(mapper, doc, false); + } + } + + public void testLongDocumentNoConstraint() throws Exception + { + byte[] doc = createBigDoc(60_000); + for (AvroMapper mapper : new AvroMapper[] { + MAPPER_VANILLA, AvroMapper.builder(new ApacheAvroFactory()).build() }) { + try (JsonParser p = mapper.reader().with(ITEMS_SCHEMA) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + } + try (JsonParser p = mapper.reader().with(ITEMS_SCHEMA).createParser(doc)) { + while (p.nextToken() != null) { } + } + } + } + + private void _testLongDocumentConstraint(AvroMapper mapper, byte[] doc, boolean stream) + throws Exception + { + // note: fixed-buffer case fails already on `createParser()` + try (JsonParser p = stream + ? mapper.reader().with(ITEMS_SCHEMA).createParser(new ByteArrayInputStream(doc)) + : mapper.reader().with(ITEMS_SCHEMA).createParser(doc)) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + final String msg = e.getMessage(); + assertTrue("unexpected message: "+msg, msg.contains("Document length (")); + assertTrue("unexpected message: "+msg, + msg.contains("exceeds the maximum allowed ("+MAX_DOC_LEN)); + } + } + + private AvroMapper constrainedMapper(boolean apacheDecoder) { + // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so + // have to construct `ApacheAvroFactory` directly + AvroFactory f = apacheDecoder ? new ApacheAvroFactory() : new AvroFactory(); + f.setStreamReadConstraints(StreamReadConstraints.builder() + .maxDocumentLength(MAX_DOC_LEN).build()); + return AvroMapper.builder(f).build(); + } + + private byte[] createBigDoc(final int size) throws Exception + { + Items items = new Items(); + // Each Item is ~50 bytes encoded; over-estimate count to be safe + for (int i = 0, len = size / 40; i < len; ++i) { + Item item = new Item(); + item.id = UUID.randomUUID().toString(); + item.size = i; + item.stuff = Long.MAX_VALUE; + items.items.add(item); + } + byte[] doc = MAPPER_VANILLA.writer(ITEMS_SCHEMA).writeValueAsBytes(items); + assertTrue("doc.length="+doc.length, doc.length > size); + return doc; + } +} diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java new file mode 100644 index 000000000..1c847036d --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java @@ -0,0 +1,90 @@ +package com.fasterxml.jackson.dataformat.avro.dos; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.apacheimpl.ApacheAvroFactory; + +// [dataformats-binary#785]: `StreamReadConstraints.maxTokenCount` for Avro +public class TokenCountAvroReadTest extends AvroTestBase +{ + public static class Ints { + public List values = new ArrayList<>(); + } + + private final AvroMapper MAPPER = newMapper(); + + private final AvroSchema SCHEMA; + { + try { + SCHEMA = MAPPER.schemaFor(Ints.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // Verify token count is tracked accurately + public void testTokenCountIsTracked() throws Exception + { + // {"values":[1,2,3]}: START_OBJECT, FIELD_NAME, START_ARRAY, + // VALUE_NUMBER_INT x3, END_ARRAY, END_OBJECT = 8 tokens + byte[] doc = createDoc(3); + for (AvroMapper mapper : new AvroMapper[] { + mapperWithMaxTokenCount(false, Long.MAX_VALUE), + mapperWithMaxTokenCount(true, Long.MAX_VALUE) }) { + try (JsonParser p = mapper.reader().with(SCHEMA).createParser(doc)) { + assertEquals(0L, p.currentTokenCount()); + while (p.nextToken() != null) { } + assertEquals(8L, p.currentTokenCount()); + } + } + } + + public void testTokenCountLimit() throws Exception + { + // createDoc(100) produces 100 + 5 tokens + byte[] doc = createDoc(100); + for (boolean apache : new boolean[] { false, true }) { + AvroMapper mapper = mapperWithMaxTokenCount(apache, 10); + _testTokenCountLimit(mapper.reader().with(SCHEMA).createParser(doc)); + _testTokenCountLimit(mapper.reader().with(SCHEMA) + .createParser(new ByteArrayInputStream(doc))); + } + } + + private void _testTokenCountLimit(JsonParser p) throws Exception + { + try { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + verifyException(e, "Token count"); + verifyException(e, "exceeds the maximum allowed (10,"); + } finally { + p.close(); + } + } + + private AvroMapper mapperWithMaxTokenCount(boolean apacheDecoder, long maxTokenCount) { + // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so + // have to construct `ApacheAvroFactory` directly + AvroFactory f = apacheDecoder ? new ApacheAvroFactory() : new AvroFactory(); + f.setStreamReadConstraints(StreamReadConstraints.builder() + .maxTokenCount(maxTokenCount).build()); + return AvroMapper.builder(f).build(); + } + + private byte[] createDoc(int numValues) throws Exception { + Ints ints = new Ints(); + for (int i = 0; i < numValues; i++) { + ints.values.add(i); + } + return MAPPER.writer(SCHEMA).writeValueAsBytes(ints); + } +} diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 4d7f3d514..88854bcbc 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -342,6 +342,9 @@ PJ Fanning (@pjfanning) (2.18.0) * Contributed #508: (avro) Ignore `specificData` field on serialization (2.18.0) + * Contributed #785: (avro) Support `StreamReadConstraints.maxDocumentLength` and + `maxTokenCount` in Avro parser + (2.18.11) Joachim Lous (@jlous) * Requested #494: Avro Schema generation: allow mapping Java Enum properties to diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 49f292d41..990f3ef98 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -14,6 +14,12 @@ Active maintainers: === Releases === ------------------------------------------------------------------------ +2.18.11 (not yet released) + +#785: (avro) Support `StreamReadConstraints.maxDocumentLength` and `maxTokenCount` + in Avro parser + (contributed by @pjfanning) + 2.18.10 (15-Aug-2026) #725: (cbor) Ensure `maxNameLength` limit enforced for CBOR parser [CVE-2026-68495] From 5865941db1e5eca072c7a7c4ad0bc270739143bd Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 18 Sep 2026 14:06:18 -0700 Subject: [PATCH 2/4] Fixes --- .../avro/deser/JacksonAvroParserImpl.java | 46 +++++++++++ .../avro/dos/LongDocumentAvroReadTest.java | 79 ++++++++++++++++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java index 0783cc171..f0029ae72 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java @@ -578,6 +578,7 @@ public void decodeString() throws IOException { } if (len > (_inputEnd - _inputPtr)) { + _validateValueLength(len); // or if not, could we read? if (len >= _inputBuffer.length) { // If not enough space, need handling similar to chunked @@ -599,6 +600,7 @@ public void skipString() throws IOException { } return; } + _validateValueLength(len); _skip(len); } @@ -789,6 +791,7 @@ public JsonToken decodeBytes() throws IOException { } _binaryValue = NO_BYTES; } else { + _validateValueLength(len); byte[] b = new byte[len]; // this is simple raw read, safe to use: _read(b, 0, len); @@ -806,12 +809,14 @@ public void skipBytes() throws IOException { } _binaryValue = NO_BYTES; } else { + _validateValueLength(len); _skip(len); } } @Override public JsonToken decodeFixed(int size) throws IOException { + _validateValueLength(size); byte[] data = new byte[size]; _read(data, 0, size); _binaryValue = data; @@ -820,9 +825,21 @@ public JsonToken decodeFixed(int size) throws IOException { @Override public void skipFixed(int size) throws IOException { + _validateValueLength(size); _skip(size); } + /** + * Helper method for checking that length of a value about to be read (or skipped) + * will not push total document length past maximum allowed: needed to avoid + * allocating (or reading) content that would fail the check in any case. + * + * @since 2.18.11 + */ + private final void _validateValueLength(long len) throws IOException { + _streamReadConstraints.validateDocumentLength(_currInputProcessed + _inputPtr + len); + } + private final void _read(byte[] target, int offset, int len) throws IOException { int ptr = _inputPtr; @@ -837,6 +854,9 @@ private final void _read(byte[] target, int offset, int len) throws IOException _inputPtr = ptr + available; offset += available; int left = len - available; + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes read directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); // and rest we can read straight from input do { int count = _inputStream.read(target, offset, left); @@ -845,6 +865,8 @@ private final void _read(byte[] target, int offset, int len) throws IOException } offset += count; left -= count; + _currInputProcessed += count; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0); } @@ -859,12 +881,17 @@ private final void _skip(int len) throws IOException } _inputPtr = _inputEnd; // mark all used, whatever it was if (_inputStream != null) { + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes skipped directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); do { int skipped = (int) _inputStream.skip(left); if (skipped < 0) { break; } left -= skipped; + _currInputProcessed += skipped; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0); } if (left > 0) { @@ -883,12 +910,17 @@ private final void _skipL(long len) throws IOException } _inputPtr = _inputEnd; // mark all used, whatever it was if (_inputStream != null) { + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes skipped directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); do { int skipped = (int) _inputStream.skip(left); if (skipped < 0) { break; } left -= skipped; + _currInputProcessed += skipped; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0L); } if (left > 0L) { @@ -1054,6 +1086,20 @@ private final void _skipByteGuaranteed() throws IOException _inputPtr += 1; } + /** + * Helper method called when the input buffer has been fully consumed and + * content will next be read (or skipped) straight from the underlying + * {@link java.io.InputStream}, bypassing the buffer: needs to flush the + * buffered byte count into {@code _currInputProcessed} so that both location + * information and document length checks remain accurate. + * + * @since 2.18.11 + */ + protected final void _markBufferConsumed() { + _currInputProcessed += _inputEnd; + _inputPtr = _inputEnd = 0; + } + protected final boolean _loadMore() throws IOException { //_currInputRowStart -= _inputEnd; diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java index 9140dcae9..c2ac1f461 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java @@ -2,7 +2,9 @@ import java.io.ByteArrayInputStream; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import com.fasterxml.jackson.core.JsonParser; @@ -25,6 +27,25 @@ public static class Items { public List items = new ArrayList<>(); } + // NOTE: Avro requires named types to match, hence same record name for both + final static String BLOB_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' },\n"+ + " { 'name':'data', 'type':'bytes' }\n"+ + " ]\n"+ + "}\n"); + + // Reader-side schema without `data`: forces writer-side `bytes` value to be skipped + final static String BLOB_NO_DATA_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + private final static int MAX_DOC_LEN = 50_000; private final AvroMapper MAPPER_VANILLA = newMapper(); @@ -64,6 +85,41 @@ public void testLongDocumentNoConstraint() throws Exception } } + // [dataformats-binary#785]: big `bytes` value is read straight from `InputStream`, + // bypassing input buffer, and must still count towards document length + public void testLongBinaryValueConstraint() throws Exception + { + byte[] doc = createBinaryDoc(200_000); + for (boolean apache : new boolean[] { false, true }) { + AvroMapper mapper = constrainedMapper(apache); + AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON); + try (JsonParser p = mapper.reader().with(schema) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException (apacheDecoder="+apache+")"); + } catch (StreamConstraintsException e) { + _verifyConstraintException(e); + } + } + } + + // Same as above but for the case where the big `bytes` value is skipped + // (writer schema has field reader schema does not) + public void testSkippedLongBinaryValueConstraint() throws Exception + { + byte[] doc = createBinaryDoc(200_000); + AvroMapper mapper = constrainedMapper(false); + AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON) + .withReaderSchema(mapper.schemaFrom(BLOB_NO_DATA_SCHEMA_JSON)); + try (JsonParser p = mapper.reader().with(schema) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + _verifyConstraintException(e); + } + } + private void _testLongDocumentConstraint(AvroMapper mapper, byte[] doc, boolean stream) throws Exception { @@ -74,13 +130,17 @@ private void _testLongDocumentConstraint(AvroMapper mapper, byte[] doc, boolean while (p.nextToken() != null) { } fail("expected StreamConstraintsException"); } catch (StreamConstraintsException e) { - final String msg = e.getMessage(); - assertTrue("unexpected message: "+msg, msg.contains("Document length (")); - assertTrue("unexpected message: "+msg, - msg.contains("exceeds the maximum allowed ("+MAX_DOC_LEN)); + _verifyConstraintException(e); } } + private void _verifyConstraintException(StreamConstraintsException e) { + final String msg = e.getMessage(); + assertTrue("unexpected message: "+msg, msg.contains("Document length (")); + assertTrue("unexpected message: "+msg, + msg.contains("exceeds the maximum allowed ("+MAX_DOC_LEN)); + } + private AvroMapper constrainedMapper(boolean apacheDecoder) { // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so // have to construct `ApacheAvroFactory` directly @@ -90,6 +150,17 @@ private AvroMapper constrainedMapper(boolean apacheDecoder) { return AvroMapper.builder(f).build(); } + private byte[] createBinaryDoc(final int payloadSize) throws Exception + { + Map blob = new LinkedHashMap<>(); + blob.put("name", "blob"); + blob.put("data", new byte[payloadSize]); + byte[] doc = MAPPER_VANILLA.writer(MAPPER_VANILLA.schemaFrom(BLOB_SCHEMA_JSON)) + .writeValueAsBytes(blob); + assertTrue("doc.length="+doc.length, doc.length > payloadSize); + return doc; + } + private byte[] createBigDoc(final int size) throws Exception { Items items = new Items(); From af574e5361b38ec105292328b1fa99533587af7e Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 18 Sep 2026 14:18:46 -0700 Subject: [PATCH 3/4] Fix constraints bug --- .../avro/apacheimpl/ApacheAvroParserImpl.java | 40 +++++++++-- .../avro/dos/LongDocumentAvroReadTest.java | 71 ++++++++++++++++--- 2 files changed, 95 insertions(+), 16 deletions(-) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java index d54e82876..3250f7e87 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java @@ -94,12 +94,15 @@ public ApacheAvroParserImpl(IOContext ctxt, int parserFeatures, int avroFeatures _bufferRecyclable = true; _apacheCodecRecycler = apacheCodecRecycler; + final boolean buffering = Feature.AVRO_BUFFERING.enabledIn(avroFeatures); // [dataformats-binary#785] Apache decoder does its own buffering, so document - // length constraint has to be applied by counting bytes it pulls from the stream + // length constraint has to be applied by counting bytes it pulls from the stream. + // But since a buffering decoder may read ahead of the actual decoding position + // by up to its buffer size, that much slack is needed to avoid false positives if (_streamReadConstraints.hasMaxDocumentLength()) { - in = new LengthCheckingInputStream(in, _streamReadConstraints); + in = new LengthCheckingInputStream(in, _streamReadConstraints, + buffering ? DECODER_FACTORY.getConfiguredBufferSize() : 0); } - final boolean buffering = Feature.AVRO_BUFFERING.enabledIn(avroFeatures); BinaryDecoder decoderToReuse = apacheCodecRecycler.acquireDecoder(); _decoder = buffering ? DECODER_FACTORY.binaryDecoder(in, decoderToReuse) @@ -429,6 +432,14 @@ protected JsonToken setString(String str) { * {@link InputStream} wrapper that applies {@link StreamReadConstraints#validateDocumentLength} * to the number of bytes read so far, for use with Apache {@link BinaryDecoder} which * reads from the stream directly. + *

+ * Note that a buffering {@link BinaryDecoder} pulls content from the stream ahead of + * the actual decoding position, by up to its buffer size: bytes that have been read + * but not (yet) decoded must not count towards document length, or a document well + * within the limit could be rejected. Since the decoder does not expose the number of + * bytes it has actually consumed, the buffer size is allowed as slack; the check is + * hence approximate (as with the non-Apache decoder), but only ever in the direction + * of allowing slightly too much. * * @since 2.18.11 */ @@ -436,18 +447,26 @@ private final static class LengthCheckingInputStream extends FilterInputStream { private final StreamReadConstraints _constraints; + /** + * Maximum number of bytes decoder may have read but not yet decoded. + */ + private final int _readAheadSlack; + private long _bytesRead; - LengthCheckingInputStream(InputStream in, StreamReadConstraints constraints) { + LengthCheckingInputStream(InputStream in, StreamReadConstraints constraints, + int readAheadSlack) { super(in); _constraints = constraints; + _readAheadSlack = readAheadSlack; } @Override public int read() throws IOException { int b = in.read(); if (b >= 0) { - _constraints.validateDocumentLength(++_bytesRead); + ++_bytesRead; + _validateLength(); } return b; } @@ -457,7 +476,7 @@ public int read(byte[] b, int off, int len) throws IOException { int count = in.read(b, off, len); if (count > 0) { _bytesRead += count; - _constraints.validateDocumentLength(_bytesRead); + _validateLength(); } return count; } @@ -467,9 +486,16 @@ public long skip(long n) throws IOException { long count = in.skip(n); if (count > 0) { _bytesRead += count; - _constraints.validateDocumentLength(_bytesRead); + _validateLength(); } return count; } + + private void _validateLength() throws IOException { + long decoded = _bytesRead - _readAheadSlack; + if (decoded > 0L) { + _constraints.validateDocumentLength(decoded); + } + } } } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java index c2ac1f461..dafe84671 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java @@ -2,6 +2,7 @@ import java.io.ByteArrayInputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -46,8 +47,28 @@ public static class Items { " ]\n"+ "}\n"); + final static String TINY_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Tiny',\n"+ + " 'fields':[\n"+ + " { 'name':'x', 'type':'int' }\n"+ + " ]\n"+ + "}\n"); + private final static int MAX_DOC_LEN = 50_000; + // Decoder variants to verify: native, Apache (buffering) and Apache (direct) + private final static int MODE_NATIVE = 0; + private final static int MODE_APACHE = 1; + private final static int MODE_APACHE_DIRECT = 2; + + // NOTE: `MODE_APACHE_DIRECT` not included: Apache `DirectBinaryDecoder.isEnd()` + // throws `UnsupportedOperationException`, so non-buffering Apache decoding does + // not currently work at all (pre-existing issue, unrelated to #785) + private final static int[] ALL_MODES = new int[] { + MODE_NATIVE, MODE_APACHE + }; + private final AvroMapper MAPPER_VANILLA = newMapper(); private final AvroSchema ITEMS_SCHEMA; @@ -63,8 +84,8 @@ public void testLongDocumentConstraint() throws Exception { // Need a bit longer than minimum since checking is approximate, not exact byte[] doc = createBigDoc(60_000); - for (boolean apache : new boolean[] { false, true }) { - AvroMapper mapper = constrainedMapper(apache); + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode); _testLongDocumentConstraint(mapper, doc, true); _testLongDocumentConstraint(mapper, doc, false); } @@ -90,13 +111,13 @@ public void testLongDocumentNoConstraint() throws Exception public void testLongBinaryValueConstraint() throws Exception { byte[] doc = createBinaryDoc(200_000); - for (boolean apache : new boolean[] { false, true }) { - AvroMapper mapper = constrainedMapper(apache); + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode); AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON); try (JsonParser p = mapper.reader().with(schema) .createParser(new ByteArrayInputStream(doc))) { while (p.nextToken() != null) { } - fail("expected StreamConstraintsException (apacheDecoder="+apache+")"); + fail("expected StreamConstraintsException (mode="+mode+")"); } catch (StreamConstraintsException e) { _verifyConstraintException(e); } @@ -108,7 +129,7 @@ public void testLongBinaryValueConstraint() throws Exception public void testSkippedLongBinaryValueConstraint() throws Exception { byte[] doc = createBinaryDoc(200_000); - AvroMapper mapper = constrainedMapper(false); + AvroMapper mapper = constrainedMapper(MODE_NATIVE); AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON) .withReaderSchema(mapper.schemaFrom(BLOB_NO_DATA_SCHEMA_JSON)); try (JsonParser p = mapper.reader().with(schema) @@ -120,6 +141,31 @@ public void testSkippedLongBinaryValueConstraint() throws Exception } } + // [dataformats-binary#785]: Apache decoder reads ahead of the decoding position, + // by up to its buffer size: must not fail a single short value read from a + // stream that holds more content past it + public void testShortValueFromLongStream() throws Exception + { + final int SHORT_MAX_DOC_LEN = 1000; + final AvroSchema schema = MAPPER_VANILLA.schemaFrom(TINY_SCHEMA_JSON); + + // Stream with lots of content, but we only read the first (1-byte) value + byte[] one = MAPPER_VANILLA.writer(schema) + .writeValueAsBytes(Collections.singletonMap("x", 42)); + byte[] many = new byte[one.length * 50_000]; + for (int i = 0; i < many.length; i += one.length) { + System.arraycopy(one, 0, many, i, one.length); + } + assertTrue("many.length="+many.length, many.length > 8 * SHORT_MAX_DOC_LEN); + + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode, SHORT_MAX_DOC_LEN); + Map value = mapper.readerFor(Map.class).with(schema) + .readValue(new ByteArrayInputStream(many)); + assertEquals("mode="+mode, 42, value.get("x")); + } + } + private void _testLongDocumentConstraint(AvroMapper mapper, byte[] doc, boolean stream) throws Exception { @@ -141,12 +187,19 @@ private void _verifyConstraintException(StreamConstraintsException e) { msg.contains("exceeds the maximum allowed ("+MAX_DOC_LEN)); } - private AvroMapper constrainedMapper(boolean apacheDecoder) { + private AvroMapper constrainedMapper(int mode) { + return constrainedMapper(mode, MAX_DOC_LEN); + } + + private AvroMapper constrainedMapper(int mode, int maxDocLen) { // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so // have to construct `ApacheAvroFactory` directly - AvroFactory f = apacheDecoder ? new ApacheAvroFactory() : new AvroFactory(); + AvroFactory f = (mode == MODE_NATIVE) ? new AvroFactory() : new ApacheAvroFactory(); + if (mode == MODE_APACHE_DIRECT) { + f.disable(AvroParser.Feature.AVRO_BUFFERING); + } f.setStreamReadConstraints(StreamReadConstraints.builder() - .maxDocumentLength(MAX_DOC_LEN).build()); + .maxDocumentLength(maxDocLen).build()); return AvroMapper.builder(f).build(); } From 529b3641f68054e43394bf02d6b9f35aae668367 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 18 Sep 2026 18:53:26 -0700 Subject: [PATCH 4/4] Improve safety of skipping content --- .../avro/deser/JacksonAvroParserImpl.java | 34 ++++++- .../schemaev/SkipNonSkippableStreamTest.java | 93 +++++++++++++++++++ .../testsupport/NonSkippingInputStream.java | 51 ++++++++++ 3 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java index f0029ae72..ba0710f40 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java @@ -870,6 +870,30 @@ private final void _read(byte[] target, int offset, int len) throws IOException } while (left > 0); } + /** + * Helper method for skipping up to specified number of bytes from the underlying + * input source. + *

+ * Note that {@link InputStream#skip} is allowed to return 0 without being at + * end-of-input -- and some implementations never skip anything -- so a zero + * return can not be taken to mean end-of-input: reading is used to both make + * progress and to find out whether end-of-input was actually reached. + * Skipped content is read in the (already consumed) input buffer and discarded. + * + * @return Number of bytes skipped; 0 or less to indicate end-of-input + * + * @since 2.18.11 + */ + private final long _skipFromInput(long left) throws IOException + { + long skipped = _inputStream.skip(left); + if (skipped > 0L) { + return skipped; + } + int toRead = (int) Math.min(left, _inputBuffer.length); + return _inputStream.read(_inputBuffer, 0, toRead); + } + private final void _skip(int len) throws IOException { int ptr = _inputPtr; @@ -885,11 +909,11 @@ private final void _skip(int len) throws IOException // input source bypass `_loadMore()` so need explicit accounting _markBufferConsumed(); do { - int skipped = (int) _inputStream.skip(left); - if (skipped < 0) { + long skipped = _skipFromInput(left); + if (skipped <= 0L) { // real end-of-input break; } - left -= skipped; + left -= (int) skipped; _currInputProcessed += skipped; _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0); @@ -914,8 +938,8 @@ private final void _skipL(long len) throws IOException // input source bypass `_loadMore()` so need explicit accounting _markBufferConsumed(); do { - int skipped = (int) _inputStream.skip(left); - if (skipped < 0) { + long skipped = _skipFromInput(left); + if (skipped <= 0L) { // real end-of-input break; } left -= skipped; diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java new file mode 100644 index 000000000..0ed697558 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java @@ -0,0 +1,93 @@ +package com.fasterxml.jackson.dataformat.avro.schemaev; + +import java.util.Map; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.testsupport.NonSkippingInputStream; + +// Test for [dataformats-binary#785]: skipping of values must work even if +// `InputStream.skip()` never skips anything +public class SkipNonSkippableStreamTest extends AvroTestBase +{ + // NOTE: Avro requires named types to match, hence same record name for both + static String SCHEMA_WITH_DATA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' },\n"+ + " { 'name':'data', 'type':'bytes' },\n"+ + " { 'name':'text', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + + // Reader schema without `data`/`text`: both writer values have to be skipped + static String SCHEMA_NO_DATA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + + private final AvroMapper MAPPER = newMapper(); + + // Values big enough not to fit in input buffer, to force skipping straight + // from the input source + private final static int BIG_VALUE_LEN = 50_000; + + public void testSkipBigValuesWithNonSkippingStream() throws Exception + { + AvroSchema writerSchema = MAPPER.schemaFrom(SCHEMA_WITH_DATA_JSON); + AvroSchema schema = writerSchema.withReaderSchema(MAPPER.schemaFrom(SCHEMA_NO_DATA_JSON)); + + byte[] doc = createDoc(writerSchema); + + try (JsonParser p = MAPPER.reader().with(schema) + .createParser(NonSkippingInputStream.wrap(doc))) { + assertToken(JsonToken.START_OBJECT, p.nextToken()); + assertToken(JsonToken.FIELD_NAME, p.nextToken()); + assertEquals("name", p.getCurrentName()); + assertToken(JsonToken.VALUE_STRING, p.nextToken()); + assertEquals("blob", p.getText()); + assertToken(JsonToken.END_OBJECT, p.nextToken()); + assertNull(p.nextToken()); + } + } + + // And also verify that true end-of-input is still reported as such + public void testTruncatedValueWithNonSkippingStream() throws Exception + { + AvroSchema writerSchema = MAPPER.schemaFrom(SCHEMA_WITH_DATA_JSON); + AvroSchema schema = writerSchema.withReaderSchema(MAPPER.schemaFrom(SCHEMA_NO_DATA_JSON)); + + byte[] doc = createDoc(writerSchema); + byte[] truncated = new byte[doc.length - (BIG_VALUE_LEN / 2)]; + System.arraycopy(doc, 0, truncated, 0, truncated.length); + + try (JsonParser p = MAPPER.reader().with(schema) + .createParser(NonSkippingInputStream.wrap(truncated))) { + while (p.nextToken() != null) { } + fail("Should not pass"); + } catch (Exception e) { + verifyException(e, "end-of-input"); + } + } + + private byte[] createDoc(AvroSchema writerSchema) throws Exception + { + Map blob = new java.util.LinkedHashMap<>(); + blob.put("name", "blob"); + blob.put("data", new byte[BIG_VALUE_LEN]); + StringBuilder sb = new StringBuilder(BIG_VALUE_LEN); + for (int i = 0; i < BIG_VALUE_LEN; ++i) { + sb.append('a'); + } + blob.put("text", sb.toString()); + byte[] doc = MAPPER.writer(writerSchema).writeValueAsBytes(blob); + assertTrue("doc.length="+doc.length, doc.length > (2 * BIG_VALUE_LEN)); + return doc; + } +} diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java new file mode 100644 index 000000000..dd8549c22 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java @@ -0,0 +1,51 @@ +package com.fasterxml.jackson.dataformat.avro.testsupport; + +import java.io.*; + +/** + * Input stream that never skips anything: {@link #skip} returns 0 even when not + * at end-of-input (something {@link InputStream#skip} is explicitly allowed to do, + * and some real-world streams do). Content must hence be consumed via {@code read()}. + *

+ * To keep a caller that does not handle this from spinning forever, an + * {@link IOException} is thrown if {@link #skip} is called many times in a row + * without any intervening read. + * + * @since 2.18.11 + */ +public class NonSkippingInputStream + extends FilterInputStream +{ + protected final static int MAX_CONSECUTIVE_SKIPS = 1000; + + protected int _consecutiveSkips; + + public NonSkippingInputStream(InputStream in) { + super(in); + } + + public static NonSkippingInputStream wrap(byte[] input) { + return new NonSkippingInputStream(new ByteArrayInputStream(input)); + } + + @Override + public long skip(long n) throws IOException { + if (++_consecutiveSkips > MAX_CONSECUTIVE_SKIPS) { + throw new IOException("skip() called "+_consecutiveSkips + +" times without intervening read(): caller not making progress"); + } + return 0L; + } + + @Override + public int read() throws IOException { + _consecutiveSkips = 0; + return in.read(); + } + + @Override + public int read(byte[] b, int offset, int len) throws IOException { + _consecutiveSkips = 0; + return in.read(b, offset, len); + } +}