From 02682041949f68e539d176fd61b0f61691823e4a Mon Sep 17 00:00:00 2001 From: Pei Yu <125331682@qq.com> Date: Fri, 21 Aug 2026 14:46:39 +0800 Subject: [PATCH] Bump Paimon version to 1.4.2 Signed-off-by: Pei Yu <125331682@qq.com> --- docker/quickstart-flink/prepare_build.sh | 20 +++++++++---------- .../encode/paimon/PaimonKeyEncoderTest.java | 4 ++-- fluss-lake/fluss-lake-paimon/pom.xml | 5 ----- .../lookup/PaimonLakeTableLookuper.java | 5 +++++ .../source/FlussArrayAsPaimonArray.java | 14 +++++++++++++ .../paimon/source/FlussRowAsPaimonRow.java | 14 +++++++++++++ .../paimon/tiering/PaimonLakeCommitter.java | 11 ++-------- .../Arrow2PaimonVectorConverter.java | 12 +++++++++++ .../src/main/resources/META-INF/NOTICE | 2 +- .../paimon/utils/PaimonConversionsTest.java | 6 +++++- pom.xml | 2 +- .../creating-a-fluss-release.mdx | 8 ++++---- .../datalake-formats/paimon.md | 8 ++++---- website/fluss-versions.json | 4 ++-- 14 files changed, 76 insertions(+), 39 deletions(-) diff --git a/docker/quickstart-flink/prepare_build.sh b/docker/quickstart-flink/prepare_build.sh index f439270f5d7..dcbbc4c0d4c 100755 --- a/docker/quickstart-flink/prepare_build.sh +++ b/docker/quickstart-flink/prepare_build.sh @@ -166,15 +166,15 @@ main() { "508255883b984483a45ca48d5af6365d4f013bb8" \ "hadoop-apache-3.3.5-2" download_jar \ - "https://repo1.maven.org/maven2/org/apache/paimon/paimon-flink-1.20/1.3.1/paimon-flink-1.20-1.3.1.jar" \ - "./paimon/paimon-flink-1.20-1.3.1.jar" \ + "https://repo1.maven.org/maven2/org/apache/paimon/paimon-flink-1.20/1.4.2/paimon-flink-1.20-1.4.2.jar" \ + "./paimon/paimon-flink-1.20-1.4.2.jar" \ "" \ - "paimon-flink-1.20-1.3.1" + "paimon-flink-1.20-1.4.2" download_jar \ - "https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/1.3.1/paimon-s3-1.3.1.jar" \ - "./paimon/paimon-s3-1.3.1.jar" \ + "https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/1.4.2/paimon-s3-1.4.2.jar" \ + "./paimon/paimon-s3-1.4.2.jar" \ "" \ - "paimon-s3-1.3.1" + "paimon-s3-1.4.2" # Iceberg-specific dependencies. These stay outside /opt/flink/lib by # default and are activated by init_iceberg.sh only when needed. @@ -231,8 +231,8 @@ verify_jars() { local paimon_jars=( "fluss-lake-paimon-*.jar" "hadoop-apache-3.3.5-2.jar" - "paimon-flink-1.20-1.3.1.jar" - "paimon-s3-1.3.1.jar" + "paimon-flink-1.20-1.4.2.jar" + "paimon-s3-1.4.2.jar" ) local iceberg_jars=( @@ -309,8 +309,8 @@ show_summary() { echo " - Base: Fluss S3 filesystem plugin" echo " - Base: Flink Faker (v0.5.3)" echo " - Paimon only: Fluss Lake Paimon connector" - echo " - Paimon only: Paimon Flink 1.20 (v1.3.1)" - echo " - Paimon only: Paimon S3 (v1.3.1)" + echo " - Paimon only: Paimon Flink 1.20 (v1.4.2)" + echo " - Paimon only: Paimon S3 (v1.4.2)" echo " - Paimon only: Hadoop Apache (v3.3.5-2)" echo " - Iceberg only: Fluss Lake Iceberg connector" echo " - Iceberg only: Iceberg Flink runtime 1.20 (v1.10.1)" diff --git a/fluss-common/src/test/java/org/apache/fluss/row/encode/paimon/PaimonKeyEncoderTest.java b/fluss-common/src/test/java/org/apache/fluss/row/encode/paimon/PaimonKeyEncoderTest.java index e770d96de5b..50adac40da2 100644 --- a/fluss-common/src/test/java/org/apache/fluss/row/encode/paimon/PaimonKeyEncoderTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/row/encode/paimon/PaimonKeyEncoderTest.java @@ -109,8 +109,8 @@ private BinaryRow genPaimonRowForAllTypes(int arity) { (int) TypeUtils.castFromString( "09:30:00.0", org.apache.fluss.types.DataTypes.TIME())); - binaryRowWriter.writeBinary(9, "1234567890".getBytes()); - binaryRowWriter.writeBinary(10, "20".getBytes()); + binaryRowWriter.writeBinary(9, "1234567890".getBytes(), 0, "1234567890".length()); + binaryRowWriter.writeBinary(10, "20".getBytes(), 0, "20".length()); binaryRowWriter.writeString(11, BinaryString.fromString("1")); binaryRowWriter.writeString(12, BinaryString.fromString("hello")); binaryRowWriter.writeDecimal(13, Decimal.fromUnscaledLong(9, 5, 2), 5); diff --git a/fluss-lake/fluss-lake-paimon/pom.xml b/fluss-lake/fluss-lake-paimon/pom.xml index e73264e76e4..65e61900bcf 100644 --- a/fluss-lake/fluss-lake-paimon/pom.xml +++ b/fluss-lake/fluss-lake-paimon/pom.xml @@ -31,11 +31,6 @@ Apache Fluss : Lake : Paimon jar - - - 1.3.1 - - org.apache.fluss diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 9e04ce15ee8..01d29aac355 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -492,6 +492,11 @@ public String[] tempDirs() { return delegate.tempDirs(); } + @Override + public String pickTempDir() { + return delegate.pickTempDir(); + } + @Override public FileIOChannel.Enumerator createChannelEnumerator() { return delegate.createChannelEnumerator(); diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussArrayAsPaimonArray.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussArrayAsPaimonArray.java index 0552bb4dea2..c815f03c849 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussArrayAsPaimonArray.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussArrayAsPaimonArray.java @@ -21,10 +21,12 @@ import org.apache.fluss.row.TimestampNtz; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalVector; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.types.ArrayType; @@ -149,6 +151,12 @@ public Variant getVariant(int pos) { "getVariant is not supported for Fluss array currently."); } + @Override + public Blob getBlob(int pos) { + throw new UnsupportedOperationException( + "getBlob is not supported for Fluss array currently."); + } + @Override public InternalArray getArray(int pos) { org.apache.fluss.row.InternalArray innerArray = flussArray.getArray(pos); @@ -158,6 +166,12 @@ public InternalArray getArray(int pos) { innerArray, ((ArrayType) elementType).getElementType()); } + @Override + public InternalVector getVector(int pos) { + throw new UnsupportedOperationException( + "getVector is not supported for Fluss array currently."); + } + @Override public InternalMap getMap(int pos) { org.apache.fluss.row.InternalMap flussMap = flussArray.getMap(pos); diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussRowAsPaimonRow.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussRowAsPaimonRow.java index 0b0b9537c4c..f20d6437ec2 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussRowAsPaimonRow.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/source/FlussRowAsPaimonRow.java @@ -21,10 +21,12 @@ import org.apache.fluss.row.TimestampNtz; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalVector; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.types.ArrayType; @@ -158,6 +160,12 @@ public Variant getVariant(int i) { "getVariant is not support for Fluss record currently."); } + @Override + public Blob getBlob(int pos) { + throw new UnsupportedOperationException( + "getBlob is not support for Fluss record currently."); + } + @Override public InternalArray getArray(int pos) { org.apache.fluss.row.InternalArray flussArray = internalRow.getArray(pos); @@ -167,6 +175,12 @@ public InternalArray getArray(int pos) { flussArray, ((ArrayType) tableRowType.getTypeAt(pos)).getElementType()); } + @Override + public InternalVector getVector(int pos) { + throw new UnsupportedOperationException( + "getVector is not support for Fluss record currently."); + } + @Override public InternalMap getMap(int pos) { org.apache.fluss.row.InternalMap flussMap = internalRow.getMap(pos); diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java index 287df30e28e..c23cc373cda 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeCommitter.java @@ -31,10 +31,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestCommittable; -import org.apache.paimon.manifest.ManifestEntry; -import org.apache.paimon.manifest.SimpleFileEntry; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.CommitCallback; @@ -290,12 +287,8 @@ private FileStoreTable getTable(TablePath tablePath, boolean isAutoSnapshotExpir public static class PaimonCommitCallback implements CommitCallback { @Override - public void call( - List baseFiles, - List deltaFiles, - List indexFiles, - Snapshot snapshot) { - currentCommitSnapshotId.set(snapshot.id()); + public void call(Context context) { + currentCommitSnapshotId.set(context.snapshot.id()); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java index a8e395725aa..d6593311f34 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java @@ -64,6 +64,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; @@ -85,6 +86,7 @@ import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; import java.util.ArrayList; import java.util.List; @@ -478,6 +480,11 @@ public Arrow2PaimonVectorConverter visit(VariantType variantType) { throw new UnsupportedOperationException(); } + @Override + public Arrow2PaimonVectorConverter visit(BlobType blobType) { + throw new UnsupportedOperationException(); + } + @Override public Arrow2PaimonVectorConverter visit(ArrayType arrayType) { final Arrow2PaimonVectorConverter arrowVectorConvertor = @@ -519,6 +526,11 @@ public ColumnVector getColumnVector() { }; } + @Override + public Arrow2PaimonVectorConverter visit(VectorType vectorType) { + throw new UnsupportedOperationException(); + } + @Override public Arrow2PaimonVectorConverter visit(MultisetType multisetType) { throw new UnsupportedOperationException("Doesn't support MultisetType."); diff --git a/fluss-lake/fluss-lake-paimon/src/main/resources/META-INF/NOTICE b/fluss-lake/fluss-lake-paimon/src/main/resources/META-INF/NOTICE index 219a6c23f04..d1ba91f598c 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/resources/META-INF/NOTICE +++ b/fluss-lake/fluss-lake-paimon/src/main/resources/META-INF/NOTICE @@ -6,4 +6,4 @@ The Apache Software Foundation (http://www.apache.org/). This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) -- org.apache.paimon:paimon-bundle:1.3.1 +- org.apache.paimon:paimon-bundle:1.4.2 diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/utils/PaimonConversionsTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/utils/PaimonConversionsTest.java index 45188b261c5..7e874ac483a 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/utils/PaimonConversionsTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/utils/PaimonConversionsTest.java @@ -66,7 +66,11 @@ void testTypesMatchFlussName() { w -> w.writeInt(0, milliOfDay), milliOfDay, DataTypeRoot.TIME_WITHOUT_TIME_ZONE); - assertMatches(DataTypes.BYTES(), w -> w.writeBinary(0, bytes), bytes, DataTypeRoot.BYTES); + assertMatches( + DataTypes.BYTES(), + w -> w.writeBinary(0, bytes, 0, bytes.length), + bytes, + DataTypeRoot.BYTES); assertMatches( DataTypes.TIMESTAMP(6), w -> w.writeTimestamp(0, Timestamp.fromEpochMillis(ms, nanos), 6), diff --git a/pom.xml b/pom.xml index ae0e5cf5820..c765972876e 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ 5.4.0 4.1.104.Final 15.0.0 - 1.3.1 + 1.4.2 1.10.1 1.1.0 1.3.0 diff --git a/website/community/how-to-release/creating-a-fluss-release.mdx b/website/community/how-to-release/creating-a-fluss-release.mdx index 16820791c0a..054663d1b2f 100644 --- a/website/community/how-to-release/creating-a-fluss-release.mdx +++ b/website/community/how-to-release/creating-a-fluss-release.mdx @@ -205,8 +205,8 @@ Next, add a new version item for the next release version in the `website/fluss- "fullVersion": "1.1-SNAPSHOT", "shortVersion": "1.1", "dockerVersion": "1.1-SNAPSHOT", - "paimonVersion": "1.3.1", - "paimonVersionShort": "1.3", + "paimonVersion": "1.4.2", + "paimonVersionShort": "1.4", "released": false } ``` @@ -219,8 +219,8 @@ Additionally, update the `fullVersion`, `shortVersion`, and `dockerVersion` fiel "fullVersion": "1.0.0", "shortVersion": "1.0", "dockerVersion": "1.0.0", - "paimonVersion": "1.3.1", - "paimonVersionShort": "1.3", + "paimonVersion": "1.4.2", + "paimonVersionShort": "1.4", "released": false } ``` diff --git a/website/docs/streaming-lakehouse/datalake-formats/paimon.md b/website/docs/streaming-lakehouse/datalake-formats/paimon.md index a859c570cca..3b223b840a5 100644 --- a/website/docs/streaming-lakehouse/datalake-formats/paimon.md +++ b/website/docs/streaming-lakehouse/datalake-formats/paimon.md @@ -32,10 +32,10 @@ Verify downloaded JARs using the [verification instructions](/downloads#verifyin ## Version Compatibility -| Use Case | Required/Tested Versions | -|-----------------|----------------------------------------------------| -| Tiering Service | Paimon **1.3** (required) | -| Union Read | Paimon 1.1, 1.2, 1.3 (tested and verified to work) | +| Use Case | Required/Tested Versions | +|-----------------|---------------------------------------------------------| +| Tiering Service | Paimon **1.4** (required) | +| Union Read | Paimon 1.1, 1.2, 1.3, 1.4 (tested and verified to work) | ## Configure Paimon as LakeHouse Storage diff --git a/website/fluss-versions.json b/website/fluss-versions.json index 13d4daabc1e..61faafd79bb 100644 --- a/website/fluss-versions.json +++ b/website/fluss-versions.json @@ -4,8 +4,8 @@ "fullVersion": "1.0-SNAPSHOT", "shortVersion": "1.0", "dockerVersion": "1.0-SNAPSHOT", - "paimonVersion": "1.3.1", - "paimonVersionShort": "1.3", + "paimonVersion": "1.4.2", + "paimonVersionShort": "1.4", "released": false }, {