From a989f2d56524536a97e9dc92c4ed63eda7f339ff Mon Sep 17 00:00:00 2001 From: Indian-boult Date: Wed, 18 Nov 2026 11:25:14 +0530 Subject: [PATCH] [client-v2] Read Dynamic values that hold an unnamed tuple The binary type encoding of a tuple without element names starts with tag 0x1F, which is also the tag of ClickHouseDataType.Tuple. readDynamicData found it in binTag2Type and used the default branch of the switch. That branch returned a bare Tuple column and left the element count and the element types in the stream, so the rest of the row was read from the wrong bytes. Send 0x1F to the tuple branch that already reads the named tuple tag 0x20. The value is now an Object[], the same as for a named tuple. Closes #3384 --- .../internal/BinaryStreamReader.java | 4 +- .../internal/BinaryStreamReaderTests.java | 31 +++++++++++ .../client/datatypes/DataTypeTests.java | 53 +++++++++++++++++++ history/latest/3384.md | 6 +++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 history/latest/3384.md diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index a3516aa8d..4089af84c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -1630,7 +1630,9 @@ private ClickHouseColumn readDynamicData() throws IOException { byte tag = readByte(); ClickHouseDataType type = ClickHouseDataType.binTag2Type.get(tag); - if (type == null) { + // The tag of an unnamed tuple is also the tag of ClickHouseDataType.Tuple, so the lookup finds it. + // Both tuple tags are decoded below, where the number of elements and their types are read. + if (type == null || type == ClickHouseDataType.Tuple) { if (tag == ClickHouseDataType.DateTime64.getBinTag() - 1) { // without timezone byte scale = readByte(); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 17cf052ad..02fc64f11 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -517,6 +517,37 @@ public void testDynamicVariantIsNotWrappedTwice() throws Exception { Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); } + @Test + public void testDynamicUnnamedTupleConsumesElementTypeEncodings() throws Exception { + // Array(Tuple(Tuple(Int32, String), Nullable(Int8))) with unnamed tuples + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.TUPLE_WITHOUT_NAMES_BIN_TAG); + BinaryStreamUtils.writeVarInt(baos, 2); + baos.write(ClickHouseDataType.TUPLE_WITHOUT_NAMES_BIN_TAG); + BinaryStreamUtils.writeVarInt(baos, 2); + baos.write(ClickHouseDataType.Int32.getBinTag()); + baos.write(ClickHouseDataType.String.getBinTag()); + baos.write(ClickHouseDataType.Nullable.getBinTag()); + baos.write(ClickHouseDataType.Int8.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt32(baos, 1); + BinaryStreamUtils.writeString(baos, "a"); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeInt8(baos, 2); + BinaryStreamUtils.writeInt32(baos, 3); + BinaryStreamUtils.writeString(baos, "b"); + BinaryStreamUtils.writeNull(baos); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(values, new Object[]{ + new Object[]{new Object[]{1, "a"}, (byte) 2}, + new Object[]{new Object[]{3, "b"}, null}}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + private static void assertEnumValue(Object actual, String expectedName, int expectedValue) { BinaryStreamReader.EnumValue value = (BinaryStreamReader.EnumValue) actual; Assert.assertEquals(value.getName(), expectedName); diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 4f68d2cb5..ab3877fc6 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -1744,6 +1744,59 @@ public void testDynamicWithNestedTypesWithQuotedNames() throws Exception { Assert.assertEquals(records.get(0).getInteger("num"), 10); } + @DataProvider(name = "dynamicUnnamedTuples") + public static Object[][] dynamicUnnamedTuples() { + return new Object[][]{ + {"Tuple(Int32, String)", "(1, 'a')", Arrays.asList(1, "a")}, + {"Tuple(Nullable(Int32), Int8)", "(1, 2)", Arrays.asList(1, (byte) 2)}, + {"Tuple(Nullable(Int32), Int8)", "(NULL, 2)", Arrays.asList(null, (byte) 2)}, + {"Tuple(Tuple(Int32, String), Int8)", "((1, 'a'), 2)", Arrays.asList(Arrays.asList(1, "a"), (byte) 2)}, + {"Tuple(a Tuple(Int32, String), b Int8)", "((1, 'a'), 2)", Arrays.asList(Arrays.asList(1, "a"), (byte) 2)}, + {"Array(Tuple(Int32, String))", "[(1, 'a'), (2, 'b')]", + Arrays.asList(Arrays.asList(1, "a"), Arrays.asList(2, "b"))}, + {"Array(Array(Tuple(Int32, String)))", "[[(1, 'a')], [(2, 'b'), (3, 'c')]]", + Arrays.asList(Arrays.asList(Arrays.asList(1, "a")), Arrays.asList(Arrays.asList(2, "b"), Arrays.asList(3, "c")))}, + {"Map(String, Tuple(Int32, String))", "map('k', (1, 'a'))", Collections.singletonMap("k", Arrays.asList(1, "a"))}, + {"Array(Variant(String, Tuple(Int32, String)))", "[(1, 'a'), 'x']", Arrays.asList(Arrays.asList(1, "a"), "x")}, + {"JSON(a Tuple(Int32, String))", "'{\"a\": [1, \"x\"]}'", Collections.singletonMap("a", Arrays.asList(1, "x"))}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "dynamicUnnamedTuples") + public void testDynamicWithUnnamedTuple(String type, String value, Object expected) throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT " + value + "::" + type + "::Dynamic AS d, 7::Int64 AS tail, dynamicType(d) AS type"); + Assert.assertEquals(records.size(), 1); + GenericRecord row = records.get(0); + Assert.assertEquals(plainValue(row.getObject("d")), expected); + Assert.assertEquals(row.getLong("tail"), 7L); + Assert.assertEquals(row.getString("type"), type); + } + + private static Object plainValue(Object value) { + if (value instanceof BinaryStreamReader.ArrayValue) { + value = ((BinaryStreamReader.ArrayValue) value).getArrayOfObjects(); + } + if (value instanceof Object[]) { + List items = new ArrayList<>(); + for (Object item : (Object[]) value) { + items.add(plainValue(item)); + } + return items; + } + if (value instanceof Map) { + Map entries = new HashMap<>(); + for (Map.Entry entry : ((Map) value).entrySet()) { + entries.put(plainValue(entry.getKey()), plainValue(entry.getValue())); + } + return entries; + } + return value; + } + @Test(groups = {"integration"}) public void testDynamicWithFixedString() throws Exception { if (isVersionMatch("(,24.8]")) { diff --git a/history/latest/3384.md b/history/latest/3384.md new file mode 100644 index 000000000..f12983ee7 --- /dev/null +++ b/history/latest/3384.md @@ -0,0 +1,6 @@ +- **[client-v2, jdbc-v2]** Fixed reading a `Dynamic` value whose type has a tuple without element names, such as + `Tuple(Int32, String)`, `Array(Tuple(Int32, String))` or `Map(String, Tuple(Int32, String))`. The reader did not + read the element types of such a tuple, so the rest of the row was read from the wrong bytes. Depending on the data, + the read failed with `Missing nested data type`, `Unsupported data type with tag 97` or `End of stream reached + before reading all data`, or it returned wrong values or no rows. Now the element types are read and the value is + returned as `Object[]`, the same as a tuple with element names. (https://github.com/ClickHouse/clickhouse-java/issues/3384)