Skip to content

branch-4.1: [fix](fe) Upgrade gRPC and remove unused LZ4 dependencies #68060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Sep 16, 2026
Merged

branch-4.1: [fix](fe) Upgrade gRPC and remove unused LZ4 dependencies #68060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions fe/be-java-extensions/max-compute-connector/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ under the License.
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>
--add-opens=java.base/java.nio=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true @{argLine}
</argLine>
<forkCount>${fe_ut_parallel}</forkCount>
<reuseForks>false</reuseForks>
<useFile>false</useFile>
Expand Down
52 changes: 52 additions & 0 deletions ...-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,68 @@

package org.apache.doris.maxcompute;

import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.table.arrow.ArrowReader;
import com.aliyun.odps.table.arrow.ArrowReaderFactory;
import com.aliyun.odps.table.arrow.ArrowWriter;
import com.aliyun.odps.table.arrow.ArrowWriterFactory;
import com.aliyun.odps.table.configuration.CompressionCodec;
import com.aliyun.odps.table.configuration.ReaderOptions;
import com.aliyun.odps.table.configuration.WriterOptions;
import com.aliyun.odps.table.enviroment.Credentials;
import com.aliyun.odps.table.enviroment.EnvironmentSettings;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.junit.Assert;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Collections;

public class MaxComputeJniWriterTest {
@Test
public void testZstdArrowRoundTrip() throws Exception {
EnvironmentSettings settings = EnvironmentSettings.newBuilder()
.withCredentials(Credentials.newBuilder()
.withAccount(new AliyunAccount("test-access-key", "test-secret-key")).build())
.build();
WriterOptions writerOptions = WriterOptions.newBuilder().withSettings(settings)
.withCompressionCodec(CompressionCodec.ZSTD).build();
try (BufferAllocator allocator = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.of(new IntVector("c", allocator))) {
IntVector vector = (IntVector) root.getVector(0);
vector.allocateNew(1024);
for (int i = 0; i < 1024; i++) {
vector.set(i, i % 7);
}
vector.setNull(5);
root.setRowCount(1024);

ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (ArrowWriter writer = ArrowWriterFactory.getRecordBatchWriter(bytes, writerOptions)) {
writer.writeBatch(root);
}

ReaderOptions readerOptions = ReaderOptions.newBuilder().withSettings(settings)
.withBufferAllocator(allocator).withCompressionCodec(CompressionCodec.ZSTD)
.withReuseBatch(true).build();
try (ArrowReader reader = ArrowReaderFactory.getRecordBatchReader(
new ByteArrayInputStream(bytes.toByteArray()), readerOptions)) {
Assert.assertTrue(reader.nextBatch());
VectorSchemaRoot decoded = reader.getCurrentValue();
Assert.assertEquals(root.getRowCount(), decoded.getRowCount());
IntVector decodedVector = (IntVector) decoded.getVector(0);
for (int i = 0; i < root.getRowCount(); i++) {
Assert.assertEquals(vector.getObject(i), decodedVector.getObject(i));
}
Assert.assertFalse(reader.nextBatch());
}
}
}

@Test
public void testPrefixBufferBytesMeasuresLeadingRowsWithoutRebuild() {
try (BufferAllocator allocator = new RootAllocator();
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,7 @@ under the License.
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql-jdbc-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
Expand Down
17 changes: 16 additions & 1 deletion fe/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ under the License.
<!-- NOTE: Using grpc-java whose version is newer than 1.34.0 will break the build on CentOS 6 due to the obsolete GLIBC -->
<grpc-java.version>1.34.0</grpc-java.version>
<!--Need to ensure that the version is the same as in arrow/java/pom.xml or compatible with it.-->
<grpc.version>1.65.1</grpc.version>
<grpc.version>1.75.0</grpc.version>
<check.freamework.version>3.53.0</check.freamework.version>
<!-- FE-only Arrow 19 startup POC: align protobuf runtime with Arrow Java 19's protobuf BOM. -->
<protobuf.version>4.33.4</protobuf.version>
Expand Down Expand Up @@ -1474,6 +1474,10 @@ under the License.
<artifactId>odps-sdk-core</artifactId>
<version>${maxcompute.version}</version>
<exclusions>
<exclusion>
<groupId>org.lz4</groupId>
<artifactId>lz4-pure-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
Expand All @@ -1492,6 +1496,17 @@ under the License.
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.odps</groupId>
<artifactId>odps-sdk-table-api</artifactId>
<version>${maxcompute.version}</version>
<exclusions>
<exclusion>
<groupId>org.lz4</groupId>
<artifactId>lz4-pure-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!-- Keep tea pinned for Alibaba SDK dependency chains that allow com.aliyun:tea:[1.1.14,2.0.0). -->
<groupId>com.aliyun</groupId>
Expand Down
Loading