From 89930bfc45adcec076867f7969aec10c054cb0a4 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 8 Sep 2026 09:49:38 +0800 Subject: [PATCH] [fix](fe) Upgrade gRPC and remove unused LZ4 dependencies (#67585) Related PR: #67065 Problem Summary: Upgrade the managed gRPC BOM from 1.65.1 to 1.75.0, matching #67065 and replacing the `grpc-netty-shaded` version affected by CVE-2025-55163. Exclude `org.lz4:lz4-pure-java` centrally from the managed `odps-sdk-core` and `odps-sdk-table-api` dependencies. Neither MaxCompute connector needs a replacement Java LZ4 dependency: the FE uses the ODPS metadata/session APIs, and the BE uses Table API Arrow readers/writers with ZSTD. The legacy SDK Tunnel LZ4 implementations are outside the active Doris data path. The BE connector retains its local Jackson exclusions. Limit `flight-sql-jdbc-driver` to test scope. Only the FE test client loads the JDBC driver; production code uses the retained `FlightServer` and `FlightSqlClient` protocol libraries. The regression framework keeps its independent JDBC driver dependency. Add a ZSTD round-trip test through the ODPS Arrow writer/reader factories, with the Java 17 direct-buffer access options required by the test JVM. This verifies 1,024 rows including a null after removing LZ4, alongside the existing isolated ODPS class-loading test. Upgrade gRPC to 1.75.0. Stop bundling the unused Java LZ4 dependency in MaxCompute connectors and the test-only Flight SQL JDBC driver in the FE runtime distribution. - Test - [ ] Regression test - [x] Unit Test - [x] Manual test (details below) Validation of the final implementation in `d9613e32c86`: - `EXTRA_FE_MODULES=maxcompute=be-java-extensions/max-compute-connector ./run-fe-ut.sh --run 'org.apache.doris.connector.maxcompute.*Test,org.apache.doris.maxcompute.*Test,!org.apache.doris.connector.maxcompute.OdpsLiveConnectivityTest'`: 146 FE and 14 BE connector tests passed, including ODPS class-loader isolation and the new ZSTD round trip. The successful test runtime classpaths were checked and contain no `net.jpountz.lz4` classes. - `build.sh --fe` passed with `DISABLE_BE_JAVA_EXTENSIONS=OFF`, `FE_MAVEN_THREADS=4`, `MVN_OPT=-Dmaven.build.cache.enabled=false`, and `--be-extension-ignore iceberg-metadata-scanner,hadoop-hudi-scanner,java-common,java-udf,jdbc-scanner,paimon-scanner,trino-connector-scanner,preload-extensions,hadoop-deps,java-writer`. All 64 reactor modules passed, including Checkstyle, FE compilation and FE/BE MaxCompute packaging. Maven wall time: 2m 31s. - Both connector dependency trees contain no Java LZ4 artifacts. The rebuilt FE plugin zip and BE connector jar contain no Java LZ4 jars/classes. The incremental output directory retained the previous build's LZ4 jar; that stale artifact was removed and the runtime plugin directory was rechecked. - `output/fe/lib` contains `grpc-netty-shaded-1.75.0.jar`; no Flight SQL JDBC driver jar is present under `output/fe`. - `git diff --check` passed. Live MaxCompute connectivity, cluster startup and regression tests were not run. (cherry picked from commit be7c013c119e4311f84794067d6a8c40e80cad17) --- .../max-compute-connector/pom.xml | 3 ++ .../maxcompute/MaxComputeJniWriterTest.java | 52 +++++++++++++++++++ fe/fe-core/pom.xml | 1 + fe/pom.xml | 17 +++++- 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/fe/be-java-extensions/max-compute-connector/pom.xml b/fe/be-java-extensions/max-compute-connector/pom.xml index 4662ab83a53c41..0fdd097e7de0d2 100644 --- a/fe/be-java-extensions/max-compute-connector/pom.xml +++ b/fe/be-java-extensions/max-compute-connector/pom.xml @@ -98,6 +98,9 @@ under the License. maven-surefire-plugin + + --add-opens=java.base/java.nio=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true @{argLine} + ${fe_ut_parallel} false false diff --git a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java index 84cc4a79d6e23d..a795637240a0bf 100644 --- a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java +++ b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java @@ -17,6 +17,16 @@ package org.apache.doris.maxcompute; +import com.aliyun.odps.account.AliyunAccount; +import com.aliyun.odps.table.arrow.ArrowReader; +import com.aliyun.odps.table.arrow.ArrowReaderFactory; +import com.aliyun.odps.table.arrow.ArrowWriter; +import com.aliyun.odps.table.arrow.ArrowWriterFactory; +import com.aliyun.odps.table.configuration.CompressionCodec; +import com.aliyun.odps.table.configuration.ReaderOptions; +import com.aliyun.odps.table.configuration.WriterOptions; +import com.aliyun.odps.table.enviroment.Credentials; +import com.aliyun.odps.table.enviroment.EnvironmentSettings; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -24,9 +34,51 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.Collections; public class MaxComputeJniWriterTest { + @Test + public void testZstdArrowRoundTrip() throws Exception { + EnvironmentSettings settings = EnvironmentSettings.newBuilder() + .withCredentials(Credentials.newBuilder() + .withAccount(new AliyunAccount("test-access-key", "test-secret-key")).build()) + .build(); + WriterOptions writerOptions = WriterOptions.newBuilder().withSettings(settings) + .withCompressionCodec(CompressionCodec.ZSTD).build(); + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.of(new IntVector("c", allocator))) { + IntVector vector = (IntVector) root.getVector(0); + vector.allocateNew(1024); + for (int i = 0; i < 1024; i++) { + vector.set(i, i % 7); + } + vector.setNull(5); + root.setRowCount(1024); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ArrowWriter writer = ArrowWriterFactory.getRecordBatchWriter(bytes, writerOptions)) { + writer.writeBatch(root); + } + + ReaderOptions readerOptions = ReaderOptions.newBuilder().withSettings(settings) + .withBufferAllocator(allocator).withCompressionCodec(CompressionCodec.ZSTD) + .withReuseBatch(true).build(); + try (ArrowReader reader = ArrowReaderFactory.getRecordBatchReader( + new ByteArrayInputStream(bytes.toByteArray()), readerOptions)) { + Assert.assertTrue(reader.nextBatch()); + VectorSchemaRoot decoded = reader.getCurrentValue(); + Assert.assertEquals(root.getRowCount(), decoded.getRowCount()); + IntVector decodedVector = (IntVector) decoded.getVector(0); + for (int i = 0; i < root.getRowCount(); i++) { + Assert.assertEquals(vector.getObject(i), decodedVector.getObject(i)); + } + Assert.assertFalse(reader.nextBatch()); + } + } + } + @Test public void testPrefixBufferBytesMeasuresLeadingRowsWithoutRebuild() { try (BufferAllocator allocator = new RootAllocator(); diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ef7b93414efe37..848a731d4ba522 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -753,6 +753,7 @@ under the License. org.apache.arrow flight-sql-jdbc-driver + test io.grpc diff --git a/fe/pom.xml b/fe/pom.xml index 4906cd982088b2..41c040cc869577 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -311,7 +311,7 @@ under the License. 1.34.0 - 1.65.1 + 1.75.0 3.53.0 4.33.4 @@ -1474,6 +1474,10 @@ under the License. odps-sdk-core ${maxcompute.version} + + org.lz4 + lz4-pure-java + org.apache.arrow arrow-vector @@ -1492,6 +1496,17 @@ under the License. + + com.aliyun.odps + odps-sdk-table-api + ${maxcompute.version} + + + org.lz4 + lz4-pure-java + + + com.aliyun