diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java index 3fd43ffa7..66da00688 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroFactory.java @@ -439,6 +439,8 @@ protected JsonParser _createParser(char[] data, int offset, int len, IOContext c @Override protected AvroParser _createParser(byte[] data, int offset, int len, IOContext ctxt) throws IOException { + // [core#1548] Validate doc length up front for fixed buffers + _streamReadConstraints.validateDocumentLength(len); // !!! 21-Apr-2017, tatu: make configurable return new JacksonAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, // return new ApacheAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java index a15bc96fd..1f6d54469 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroFactory.java @@ -51,6 +51,8 @@ protected AvroParser _createParser(InputStream in, IOContext ctxt) throws IOExce @Override protected AvroParser _createParser(byte[] data, int offset, int len, IOContext ctxt) throws IOException { + // [core#1548] Validate doc length up front for fixed buffers + _streamReadConstraints.validateDocumentLength(len); return new ApacheAvroParserImpl(ctxt, _parserFeatures, _avroParserFeatures, _avroRecyclerPool.acquireAndLinkPooled(), _objectCodec, data, offset, len); diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java index ab59a3979..3250f7e87 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/apacheimpl/ApacheAvroParserImpl.java @@ -1,5 +1,6 @@ package com.fasterxml.jackson.dataformat.avro.apacheimpl; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Writer; @@ -94,6 +95,14 @@ public ApacheAvroParserImpl(IOContext ctxt, int parserFeatures, int avroFeatures _apacheCodecRecycler = apacheCodecRecycler; final boolean buffering = Feature.AVRO_BUFFERING.enabledIn(avroFeatures); + // [dataformats-binary#785] Apache decoder does its own buffering, so document + // length constraint has to be applied by counting bytes it pulls from the stream. + // But since a buffering decoder may read ahead of the actual decoding position + // by up to its buffer size, that much slack is needed to avoid false positives + if (_streamReadConstraints.hasMaxDocumentLength()) { + in = new LengthCheckingInputStream(in, _streamReadConstraints, + buffering ? DECODER_FACTORY.getConfiguredBufferSize() : 0); + } BinaryDecoder decoderToReuse = apacheCodecRecycler.acquireDecoder(); _decoder = buffering ? DECODER_FACTORY.binaryDecoder(in, decoderToReuse) @@ -412,4 +421,81 @@ protected JsonToken setString(String str) { _textValue = str; return JsonToken.VALUE_STRING; } + + /* + /********************************************************** + /* Helper classes + /********************************************************** + */ + + /** + * {@link InputStream} wrapper that applies {@link StreamReadConstraints#validateDocumentLength} + * to the number of bytes read so far, for use with Apache {@link BinaryDecoder} which + * reads from the stream directly. + *

+ * Note that a buffering {@link BinaryDecoder} pulls content from the stream ahead of + * the actual decoding position, by up to its buffer size: bytes that have been read + * but not (yet) decoded must not count towards document length, or a document well + * within the limit could be rejected. Since the decoder does not expose the number of + * bytes it has actually consumed, the buffer size is allowed as slack; the check is + * hence approximate (as with the non-Apache decoder), but only ever in the direction + * of allowing slightly too much. + * + * @since 2.18.11 + */ + private final static class LengthCheckingInputStream extends FilterInputStream + { + private final StreamReadConstraints _constraints; + + /** + * Maximum number of bytes decoder may have read but not yet decoded. + */ + private final int _readAheadSlack; + + private long _bytesRead; + + LengthCheckingInputStream(InputStream in, StreamReadConstraints constraints, + int readAheadSlack) { + super(in); + _constraints = constraints; + _readAheadSlack = readAheadSlack; + } + + @Override + public int read() throws IOException { + int b = in.read(); + if (b >= 0) { + ++_bytesRead; + _validateLength(); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int count = in.read(b, off, len); + if (count > 0) { + _bytesRead += count; + _validateLength(); + } + return count; + } + + @Override + public long skip(long n) throws IOException { + long count = in.skip(n); + if (count > 0) { + _bytesRead += count; + _validateLength(); + } + return count; + } + + private void _validateLength() throws IOException { + long decoded = _bytesRead - _readAheadSlack; + if (decoded > 0L) { + _constraints.validateDocumentLength(decoded); + } + } + } } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java index d54893503..b402a8707 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java @@ -95,8 +95,7 @@ public JsonToken nextToken() throws IOException _enumIndex = -1; _binaryValue = null; JsonToken t = _avroContext.nextToken(); - _currToken = t; - return t; + return _nullSafeUpdateToken(t); } @Override diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java index e3f74313f..ba0710f40 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/JacksonAvroParserImpl.java @@ -578,6 +578,7 @@ public void decodeString() throws IOException { } if (len > (_inputEnd - _inputPtr)) { + _validateValueLength(len); // or if not, could we read? if (len >= _inputBuffer.length) { // If not enough space, need handling similar to chunked @@ -599,6 +600,7 @@ public void skipString() throws IOException { } return; } + _validateValueLength(len); _skip(len); } @@ -789,6 +791,7 @@ public JsonToken decodeBytes() throws IOException { } _binaryValue = NO_BYTES; } else { + _validateValueLength(len); byte[] b = new byte[len]; // this is simple raw read, safe to use: _read(b, 0, len); @@ -806,12 +809,14 @@ public void skipBytes() throws IOException { } _binaryValue = NO_BYTES; } else { + _validateValueLength(len); _skip(len); } } @Override public JsonToken decodeFixed(int size) throws IOException { + _validateValueLength(size); byte[] data = new byte[size]; _read(data, 0, size); _binaryValue = data; @@ -820,9 +825,21 @@ public JsonToken decodeFixed(int size) throws IOException { @Override public void skipFixed(int size) throws IOException { + _validateValueLength(size); _skip(size); } + /** + * Helper method for checking that length of a value about to be read (or skipped) + * will not push total document length past maximum allowed: needed to avoid + * allocating (or reading) content that would fail the check in any case. + * + * @since 2.18.11 + */ + private final void _validateValueLength(long len) throws IOException { + _streamReadConstraints.validateDocumentLength(_currInputProcessed + _inputPtr + len); + } + private final void _read(byte[] target, int offset, int len) throws IOException { int ptr = _inputPtr; @@ -837,6 +854,9 @@ private final void _read(byte[] target, int offset, int len) throws IOException _inputPtr = ptr + available; offset += available; int left = len - available; + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes read directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); // and rest we can read straight from input do { int count = _inputStream.read(target, offset, left); @@ -845,9 +865,35 @@ private final void _read(byte[] target, int offset, int len) throws IOException } offset += count; left -= count; + _currInputProcessed += count; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0); } + /** + * Helper method for skipping up to specified number of bytes from the underlying + * input source. + *

+ * Note that {@link InputStream#skip} is allowed to return 0 without being at + * end-of-input -- and some implementations never skip anything -- so a zero + * return can not be taken to mean end-of-input: reading is used to both make + * progress and to find out whether end-of-input was actually reached. + * Skipped content is read in the (already consumed) input buffer and discarded. + * + * @return Number of bytes skipped; 0 or less to indicate end-of-input + * + * @since 2.18.11 + */ + private final long _skipFromInput(long left) throws IOException + { + long skipped = _inputStream.skip(left); + if (skipped > 0L) { + return skipped; + } + int toRead = (int) Math.min(left, _inputBuffer.length); + return _inputStream.read(_inputBuffer, 0, toRead); + } + private final void _skip(int len) throws IOException { int ptr = _inputPtr; @@ -859,12 +905,17 @@ private final void _skip(int len) throws IOException } _inputPtr = _inputEnd; // mark all used, whatever it was if (_inputStream != null) { + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes skipped directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); do { - int skipped = (int) _inputStream.skip(left); - if (skipped < 0) { + long skipped = _skipFromInput(left); + if (skipped <= 0L) { // real end-of-input break; } - left -= skipped; + left -= (int) skipped; + _currInputProcessed += skipped; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0); } if (left > 0) { @@ -883,12 +934,17 @@ private final void _skipL(long len) throws IOException } _inputPtr = _inputEnd; // mark all used, whatever it was if (_inputStream != null) { + // 18-Sep-2026, tatu: [dataformats-binary#785] Bytes skipped directly from + // input source bypass `_loadMore()` so need explicit accounting + _markBufferConsumed(); do { - int skipped = (int) _inputStream.skip(left); - if (skipped < 0) { + long skipped = _skipFromInput(left); + if (skipped <= 0L) { // real end-of-input break; } left -= skipped; + _currInputProcessed += skipped; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); } while (left > 0L); } if (left > 0L) { @@ -1054,12 +1110,27 @@ private final void _skipByteGuaranteed() throws IOException _inputPtr += 1; } + /** + * Helper method called when the input buffer has been fully consumed and + * content will next be read (or skipped) straight from the underlying + * {@link java.io.InputStream}, bypassing the buffer: needs to flush the + * buffered byte count into {@code _currInputProcessed} so that both location + * information and document length checks remain accurate. + * + * @since 2.18.11 + */ + protected final void _markBufferConsumed() { + _currInputProcessed += _inputEnd; + _inputPtr = _inputEnd = 0; + } + protected final boolean _loadMore() throws IOException { //_currInputRowStart -= _inputEnd; if (_inputStream != null) { int count = _inputStream.read(_inputBuffer, 0, _inputBuffer.length); _currInputProcessed += _inputEnd; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); _inputPtr = 0; if (count > 0) { _inputEnd = count; @@ -1086,6 +1157,7 @@ protected final void _loadToHaveAtLeast(int minAvailable) throws IOException // Need to move remaining data in front? int amount = _inputEnd - _inputPtr; _currInputProcessed += _inputPtr; + _streamReadConstraints.validateDocumentLength(_currInputProcessed); if (_inputPtr > 0) { if (amount > 0) { //_currInputRowStart -= _inputPtr; diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java new file mode 100644 index 000000000..dafe84671 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/LongDocumentAvroReadTest.java @@ -0,0 +1,232 @@ +package com.fasterxml.jackson.dataformat.avro.dos; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.apacheimpl.ApacheAvroFactory; + +// [dataformats-binary#785]: `StreamReadConstraints.maxDocumentLength` for Avro +public class LongDocumentAvroReadTest extends AvroTestBase +{ + public static class Item { + public String id; + public int size; + public long stuff; + } + + public static class Items { + public List items = new ArrayList<>(); + } + + // NOTE: Avro requires named types to match, hence same record name for both + final static String BLOB_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' },\n"+ + " { 'name':'data', 'type':'bytes' }\n"+ + " ]\n"+ + "}\n"); + + // Reader-side schema without `data`: forces writer-side `bytes` value to be skipped + final static String BLOB_NO_DATA_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + + final static String TINY_SCHEMA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Tiny',\n"+ + " 'fields':[\n"+ + " { 'name':'x', 'type':'int' }\n"+ + " ]\n"+ + "}\n"); + + private final static int MAX_DOC_LEN = 50_000; + + // Decoder variants to verify: native, Apache (buffering) and Apache (direct) + private final static int MODE_NATIVE = 0; + private final static int MODE_APACHE = 1; + private final static int MODE_APACHE_DIRECT = 2; + + // NOTE: `MODE_APACHE_DIRECT` not included: Apache `DirectBinaryDecoder.isEnd()` + // throws `UnsupportedOperationException`, so non-buffering Apache decoding does + // not currently work at all (pre-existing issue, unrelated to #785) + private final static int[] ALL_MODES = new int[] { + MODE_NATIVE, MODE_APACHE + }; + + private final AvroMapper MAPPER_VANILLA = newMapper(); + + private final AvroSchema ITEMS_SCHEMA; + { + try { + ITEMS_SCHEMA = MAPPER_VANILLA.schemaFor(Items.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void testLongDocumentConstraint() throws Exception + { + // Need a bit longer than minimum since checking is approximate, not exact + byte[] doc = createBigDoc(60_000); + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode); + _testLongDocumentConstraint(mapper, doc, true); + _testLongDocumentConstraint(mapper, doc, false); + } + } + + public void testLongDocumentNoConstraint() throws Exception + { + byte[] doc = createBigDoc(60_000); + for (AvroMapper mapper : new AvroMapper[] { + MAPPER_VANILLA, AvroMapper.builder(new ApacheAvroFactory()).build() }) { + try (JsonParser p = mapper.reader().with(ITEMS_SCHEMA) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + } + try (JsonParser p = mapper.reader().with(ITEMS_SCHEMA).createParser(doc)) { + while (p.nextToken() != null) { } + } + } + } + + // [dataformats-binary#785]: big `bytes` value is read straight from `InputStream`, + // bypassing input buffer, and must still count towards document length + public void testLongBinaryValueConstraint() throws Exception + { + byte[] doc = createBinaryDoc(200_000); + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode); + AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON); + try (JsonParser p = mapper.reader().with(schema) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException (mode="+mode+")"); + } catch (StreamConstraintsException e) { + _verifyConstraintException(e); + } + } + } + + // Same as above but for the case where the big `bytes` value is skipped + // (writer schema has field reader schema does not) + public void testSkippedLongBinaryValueConstraint() throws Exception + { + byte[] doc = createBinaryDoc(200_000); + AvroMapper mapper = constrainedMapper(MODE_NATIVE); + AvroSchema schema = mapper.schemaFrom(BLOB_SCHEMA_JSON) + .withReaderSchema(mapper.schemaFrom(BLOB_NO_DATA_SCHEMA_JSON)); + try (JsonParser p = mapper.reader().with(schema) + .createParser(new ByteArrayInputStream(doc))) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + _verifyConstraintException(e); + } + } + + // [dataformats-binary#785]: Apache decoder reads ahead of the decoding position, + // by up to its buffer size: must not fail a single short value read from a + // stream that holds more content past it + public void testShortValueFromLongStream() throws Exception + { + final int SHORT_MAX_DOC_LEN = 1000; + final AvroSchema schema = MAPPER_VANILLA.schemaFrom(TINY_SCHEMA_JSON); + + // Stream with lots of content, but we only read the first (1-byte) value + byte[] one = MAPPER_VANILLA.writer(schema) + .writeValueAsBytes(Collections.singletonMap("x", 42)); + byte[] many = new byte[one.length * 50_000]; + for (int i = 0; i < many.length; i += one.length) { + System.arraycopy(one, 0, many, i, one.length); + } + assertTrue("many.length="+many.length, many.length > 8 * SHORT_MAX_DOC_LEN); + + for (int mode : ALL_MODES) { + AvroMapper mapper = constrainedMapper(mode, SHORT_MAX_DOC_LEN); + Map value = mapper.readerFor(Map.class).with(schema) + .readValue(new ByteArrayInputStream(many)); + assertEquals("mode="+mode, 42, value.get("x")); + } + } + + private void _testLongDocumentConstraint(AvroMapper mapper, byte[] doc, boolean stream) + throws Exception + { + // note: fixed-buffer case fails already on `createParser()` + try (JsonParser p = stream + ? mapper.reader().with(ITEMS_SCHEMA).createParser(new ByteArrayInputStream(doc)) + : mapper.reader().with(ITEMS_SCHEMA).createParser(doc)) { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + _verifyConstraintException(e); + } + } + + private void _verifyConstraintException(StreamConstraintsException e) { + final String msg = e.getMessage(); + assertTrue("unexpected message: "+msg, msg.contains("Document length (")); + assertTrue("unexpected message: "+msg, + msg.contains("exceeds the maximum allowed ("+MAX_DOC_LEN)); + } + + private AvroMapper constrainedMapper(int mode) { + return constrainedMapper(mode, MAX_DOC_LEN); + } + + private AvroMapper constrainedMapper(int mode, int maxDocLen) { + // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so + // have to construct `ApacheAvroFactory` directly + AvroFactory f = (mode == MODE_NATIVE) ? new AvroFactory() : new ApacheAvroFactory(); + if (mode == MODE_APACHE_DIRECT) { + f.disable(AvroParser.Feature.AVRO_BUFFERING); + } + f.setStreamReadConstraints(StreamReadConstraints.builder() + .maxDocumentLength(maxDocLen).build()); + return AvroMapper.builder(f).build(); + } + + private byte[] createBinaryDoc(final int payloadSize) throws Exception + { + Map blob = new LinkedHashMap<>(); + blob.put("name", "blob"); + blob.put("data", new byte[payloadSize]); + byte[] doc = MAPPER_VANILLA.writer(MAPPER_VANILLA.schemaFrom(BLOB_SCHEMA_JSON)) + .writeValueAsBytes(blob); + assertTrue("doc.length="+doc.length, doc.length > payloadSize); + return doc; + } + + private byte[] createBigDoc(final int size) throws Exception + { + Items items = new Items(); + // Each Item is ~50 bytes encoded; over-estimate count to be safe + for (int i = 0, len = size / 40; i < len; ++i) { + Item item = new Item(); + item.id = UUID.randomUUID().toString(); + item.size = i; + item.stuff = Long.MAX_VALUE; + items.items.add(item); + } + byte[] doc = MAPPER_VANILLA.writer(ITEMS_SCHEMA).writeValueAsBytes(items); + assertTrue("doc.length="+doc.length, doc.length > size); + return doc; + } +} diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java new file mode 100644 index 000000000..1c847036d --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/dos/TokenCountAvroReadTest.java @@ -0,0 +1,90 @@ +package com.fasterxml.jackson.dataformat.avro.dos; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.apacheimpl.ApacheAvroFactory; + +// [dataformats-binary#785]: `StreamReadConstraints.maxTokenCount` for Avro +public class TokenCountAvroReadTest extends AvroTestBase +{ + public static class Ints { + public List values = new ArrayList<>(); + } + + private final AvroMapper MAPPER = newMapper(); + + private final AvroSchema SCHEMA; + { + try { + SCHEMA = MAPPER.schemaFor(Ints.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // Verify token count is tracked accurately + public void testTokenCountIsTracked() throws Exception + { + // {"values":[1,2,3]}: START_OBJECT, FIELD_NAME, START_ARRAY, + // VALUE_NUMBER_INT x3, END_ARRAY, END_OBJECT = 8 tokens + byte[] doc = createDoc(3); + for (AvroMapper mapper : new AvroMapper[] { + mapperWithMaxTokenCount(false, Long.MAX_VALUE), + mapperWithMaxTokenCount(true, Long.MAX_VALUE) }) { + try (JsonParser p = mapper.reader().with(SCHEMA).createParser(doc)) { + assertEquals(0L, p.currentTokenCount()); + while (p.nextToken() != null) { } + assertEquals(8L, p.currentTokenCount()); + } + } + } + + public void testTokenCountLimit() throws Exception + { + // createDoc(100) produces 100 + 5 tokens + byte[] doc = createDoc(100); + for (boolean apache : new boolean[] { false, true }) { + AvroMapper mapper = mapperWithMaxTokenCount(apache, 10); + _testTokenCountLimit(mapper.reader().with(SCHEMA).createParser(doc)); + _testTokenCountLimit(mapper.reader().with(SCHEMA) + .createParser(new ByteArrayInputStream(doc))); + } + } + + private void _testTokenCountLimit(JsonParser p) throws Exception + { + try { + while (p.nextToken() != null) { } + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + verifyException(e, "Token count"); + verifyException(e, "exceeds the maximum allowed (10,"); + } finally { + p.close(); + } + } + + private AvroMapper mapperWithMaxTokenCount(boolean apacheDecoder, long maxTokenCount) { + // 2.18 `AvroFactoryBuilder` does not (yet) honor Apache decoder setting, so + // have to construct `ApacheAvroFactory` directly + AvroFactory f = apacheDecoder ? new ApacheAvroFactory() : new AvroFactory(); + f.setStreamReadConstraints(StreamReadConstraints.builder() + .maxTokenCount(maxTokenCount).build()); + return AvroMapper.builder(f).build(); + } + + private byte[] createDoc(int numValues) throws Exception { + Ints ints = new Ints(); + for (int i = 0; i < numValues; i++) { + ints.values.add(i); + } + return MAPPER.writer(SCHEMA).writeValueAsBytes(ints); + } +} diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java new file mode 100644 index 000000000..0ed697558 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schemaev/SkipNonSkippableStreamTest.java @@ -0,0 +1,93 @@ +package com.fasterxml.jackson.dataformat.avro.schemaev; + +import java.util.Map; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import com.fasterxml.jackson.dataformat.avro.*; +import com.fasterxml.jackson.dataformat.avro.testsupport.NonSkippingInputStream; + +// Test for [dataformats-binary#785]: skipping of values must work even if +// `InputStream.skip()` never skips anything +public class SkipNonSkippableStreamTest extends AvroTestBase +{ + // NOTE: Avro requires named types to match, hence same record name for both + static String SCHEMA_WITH_DATA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' },\n"+ + " { 'name':'data', 'type':'bytes' },\n"+ + " { 'name':'text', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + + // Reader schema without `data`/`text`: both writer values have to be skipped + static String SCHEMA_NO_DATA_JSON = aposToQuotes("{\n"+ + " 'type':'record',\n"+ + " 'name':'Blob',\n"+ + " 'fields':[\n"+ + " { 'name':'name', 'type':'string' }\n"+ + " ]\n"+ + "}\n"); + + private final AvroMapper MAPPER = newMapper(); + + // Values big enough not to fit in input buffer, to force skipping straight + // from the input source + private final static int BIG_VALUE_LEN = 50_000; + + public void testSkipBigValuesWithNonSkippingStream() throws Exception + { + AvroSchema writerSchema = MAPPER.schemaFrom(SCHEMA_WITH_DATA_JSON); + AvroSchema schema = writerSchema.withReaderSchema(MAPPER.schemaFrom(SCHEMA_NO_DATA_JSON)); + + byte[] doc = createDoc(writerSchema); + + try (JsonParser p = MAPPER.reader().with(schema) + .createParser(NonSkippingInputStream.wrap(doc))) { + assertToken(JsonToken.START_OBJECT, p.nextToken()); + assertToken(JsonToken.FIELD_NAME, p.nextToken()); + assertEquals("name", p.getCurrentName()); + assertToken(JsonToken.VALUE_STRING, p.nextToken()); + assertEquals("blob", p.getText()); + assertToken(JsonToken.END_OBJECT, p.nextToken()); + assertNull(p.nextToken()); + } + } + + // And also verify that true end-of-input is still reported as such + public void testTruncatedValueWithNonSkippingStream() throws Exception + { + AvroSchema writerSchema = MAPPER.schemaFrom(SCHEMA_WITH_DATA_JSON); + AvroSchema schema = writerSchema.withReaderSchema(MAPPER.schemaFrom(SCHEMA_NO_DATA_JSON)); + + byte[] doc = createDoc(writerSchema); + byte[] truncated = new byte[doc.length - (BIG_VALUE_LEN / 2)]; + System.arraycopy(doc, 0, truncated, 0, truncated.length); + + try (JsonParser p = MAPPER.reader().with(schema) + .createParser(NonSkippingInputStream.wrap(truncated))) { + while (p.nextToken() != null) { } + fail("Should not pass"); + } catch (Exception e) { + verifyException(e, "end-of-input"); + } + } + + private byte[] createDoc(AvroSchema writerSchema) throws Exception + { + Map blob = new java.util.LinkedHashMap<>(); + blob.put("name", "blob"); + blob.put("data", new byte[BIG_VALUE_LEN]); + StringBuilder sb = new StringBuilder(BIG_VALUE_LEN); + for (int i = 0; i < BIG_VALUE_LEN; ++i) { + sb.append('a'); + } + blob.put("text", sb.toString()); + byte[] doc = MAPPER.writer(writerSchema).writeValueAsBytes(blob); + assertTrue("doc.length="+doc.length, doc.length > (2 * BIG_VALUE_LEN)); + return doc; + } +} diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java new file mode 100644 index 000000000..dd8549c22 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/testsupport/NonSkippingInputStream.java @@ -0,0 +1,51 @@ +package com.fasterxml.jackson.dataformat.avro.testsupport; + +import java.io.*; + +/** + * Input stream that never skips anything: {@link #skip} returns 0 even when not + * at end-of-input (something {@link InputStream#skip} is explicitly allowed to do, + * and some real-world streams do). Content must hence be consumed via {@code read()}. + *

+ * To keep a caller that does not handle this from spinning forever, an + * {@link IOException} is thrown if {@link #skip} is called many times in a row + * without any intervening read. + * + * @since 2.18.11 + */ +public class NonSkippingInputStream + extends FilterInputStream +{ + protected final static int MAX_CONSECUTIVE_SKIPS = 1000; + + protected int _consecutiveSkips; + + public NonSkippingInputStream(InputStream in) { + super(in); + } + + public static NonSkippingInputStream wrap(byte[] input) { + return new NonSkippingInputStream(new ByteArrayInputStream(input)); + } + + @Override + public long skip(long n) throws IOException { + if (++_consecutiveSkips > MAX_CONSECUTIVE_SKIPS) { + throw new IOException("skip() called "+_consecutiveSkips + +" times without intervening read(): caller not making progress"); + } + return 0L; + } + + @Override + public int read() throws IOException { + _consecutiveSkips = 0; + return in.read(); + } + + @Override + public int read(byte[] b, int offset, int len) throws IOException { + _consecutiveSkips = 0; + return in.read(b, offset, len); + } +} diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index a2c94a77d..595d87ffa 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -345,6 +345,9 @@ PJ Fanning (@pjfanning) * Contributed #783: (protobuf) Support `StreamReadConstraints.maxDocumentLength` in `ProtobufParser` (2.18.11) + * Contributed #785: (avro) Support `StreamReadConstraints.maxDocumentLength` and + `maxTokenCount` in Avro parser + (2.18.11) Joachim Lous (@jlous) * Requested #494: Avro Schema generation: allow mapping Java Enum properties to diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 4f1c5655b..7b3a2d507 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -18,6 +18,9 @@ Active maintainers: #783: (protobuf) Support `StreamReadConstraints.maxDocumentLength` in `ProtobufParser` (contributed by @pjfanning) +#785: (avro) Support `StreamReadConstraints.maxDocumentLength` and `maxTokenCount` + in Avro parser + (contributed by @pjfanning) 2.18.10 (15-Aug-2026)