From 3d8471fa901a4cb17433e992a0a7b9351811ad7a Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 1 Oct 2026 13:13:18 +0800 Subject: [PATCH 1/2] [fix](branch-4.2) Repair index regression and BE unit tests Pin common expression pushdown in gram and SNII suites so fuzzy session defaults cannot bypass their index checks. Include peer I/O counters in file-cache deltas and initialize the candidate bitmap in the SegmentIterator test fixture. --- be/src/io/cache/block_file_cache_profile.cpp | 10 ++++++++++ .../segment/segment_iterator_apply_index_expr_test.cpp | 1 + .../inverted_index_p0/gram/test_gram_compaction.groovy | 2 ++ .../gram/test_gram_index_order.groovy | 2 ++ .../gram/test_gram_metadata_inherit.groovy | 2 ++ .../gram/test_gram_pattern_recall.groovy | 2 ++ .../gram/test_gram_policy_recovery.groovy | 2 ++ .../gram/test_gram_regexp_like.groovy | 2 ++ .../gram/test_gram_schema_change.groovy | 2 ++ .../inverted_index_p0/gram/test_gram_stop_gram.groovy | 2 ++ .../storage_format/test_storage_format_snii.groovy | 2 ++ .../test_storage_format_snii_custom_analyzer.groovy | 2 ++ .../test_storage_format_snii_norms.groovy | 2 ++ .../test_storage_format_snii_utf8_wildcard.groovy | 2 ++ .../test_variant_search_subcolumn_snii.groovy | 2 ++ .../v2/with_index/test_variant_v2_snii_index.groovy | 2 ++ .../with_index/test_variant_snii_compaction.groovy | 2 ++ .../suites/variant_p0/with_index/var_index_snii.groovy | 2 ++ 18 files changed, 43 insertions(+) diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index d63ff9cf3a2f1a..a15dd6e5b4c34a 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -143,6 +143,16 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(segment_footer_index_bytes_write_into_cache); SUBTRACT_FIELD(remote_only_on_miss_triggered); SUBTRACT_FIELD(remote_only_on_miss_threshold_bytes); + SUBTRACT_FIELD(num_cross_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_cross_cg_peer); + SUBTRACT_FIELD(cross_cg_peer_io_timer); + SUBTRACT_FIELD(num_same_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_same_cg_peer); + SUBTRACT_FIELD(same_cg_peer_io_timer); + SUBTRACT_FIELD(num_peer_race_peer_win); + SUBTRACT_FIELD(num_peer_race_s3_win); + SUBTRACT_FIELD(num_peer_lazy_fetch); + SUBTRACT_FIELD(peer_lazy_fetch_timer); #undef SUBTRACT_FIELD return diff; } diff --git a/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp b/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp index 09d9f0e4a80dc1..84d012e8ead2bb 100644 --- a/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp +++ b/be/test/storage/segment/segment_iterator_apply_index_expr_test.cpp @@ -127,6 +127,7 @@ class SegmentIteratorApplyIndexExprTest : public testing::Test { } _read_schema = std::make_shared(_tablet_schema->columns(), read_column_ids); _iter = std::make_unique(_segment, _read_schema); + _iter->_row_bitmap.addRange(0, _segment->num_rows()); // Set up RuntimeState with fallback enabled so _downgrade_without_index works TQueryOptions query_options; diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy index 2b5014d655cba4..b8dc5db6ec562d 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_compaction.groovy @@ -18,6 +18,8 @@ // Compaction rebuilds docs-only gram indexes from merged rows. Verify query results // and index availability before and after compaction. suite("test_gram_compaction", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def waitAnalyzerInstalled = { String name -> def deadline = System.currentTimeMillis() + 180_000 Exception lastNotFound = null diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy index 244ae38ccf5f3b..1aee562d70f4cd 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_index_order.groovy @@ -28,6 +28,8 @@ import org.apache.doris.regression.action.ProfileAction // whichever tokenized index came first. A docs-only index in that place -- and a gram index // is docs-only by default -- made a valid query fail. suite("test_gram_index_order", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def sparseTok = "gram_order_sparse_tok" def sparseAna = "gram_order_sparse" def positionalAna = "gram_order_positional" diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy index fe64b4ecb97b32..59ce906ddfdc5a 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_metadata_inherit.groovy @@ -20,6 +20,8 @@ import java.util.regex.Pattern import org.apache.doris.regression.action.ProfileAction suite("test_gram_metadata_inherit", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" // Cloud does not support the named-index BUILD syntax used here. Keep this case scoped // to adding one requested index while IndexBuilder inherits the unchanged gram indexes. if (isCloudMode()) { diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy index cfe90bfd16ee87..23918556be6c2c 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_pattern_recall.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_gram_pattern_recall", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql "DROP INVERTED INDEX TOKENIZER IF EXISTS gram_recall_invalid_tokenizer" test { sql """CREATE INVERTED INDEX TOKENIZER gram_recall_invalid_tokenizer diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy index 6d9374db849203..f254b843deaf7d 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_policy_recovery.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_gram_policy_recovery", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql "SET enable_sql_cache=false" // The condition cache must be off too: gram deliberately keeps its LIKE / REGEXP expression // in _common_expr_ctxs_push_down for the row-level recheck, so the segment iterator never diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy index 26b5bd3a6f476c..469638a4b3eccd 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_regexp_like.groovy @@ -22,6 +22,8 @@ import org.apache.doris.regression.action.ProfileAction // Compare LIKE and REGEXP results with and without gram index pushdown. // The original expression must still decide which candidate rows match. suite("test_gram_regexp_like", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def tbl = "t_gram_regexp_like" // Policy names are globally unique: tokenizers and analyzers share one namespace, so the two // names must differ diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy index 8cef9fc3d83399..54cf16e66ea3b9 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_schema_change.groovy @@ -22,6 +22,8 @@ import java.util.regex.Pattern // Check that gram indexes remain readable and useful after schema changes and new loads. // LIKE and REGEXP results must match a scan at every step. suite("test_gram_schema_change", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def waitAnalyzerInstalled = { String name -> def deadline = System.currentTimeMillis() + 180_000 Exception lastNotFound = null diff --git a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy index a3f8c7b0cba95a..a16db778a9bc9c 100644 --- a/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy +++ b/regression-test/suites/inverted_index_p0/gram/test_gram_stop_gram.groovy @@ -22,6 +22,8 @@ import java.util.regex.Pattern // Compare one large segment that drops common postings with smaller segments that keep them. // Both layouts must return the same LIKE and REGEXP results as a scan. suite("test_gram_stop_gram", "p0") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def waitAnalyzerInstalled = { String name -> def deadline = System.currentTimeMillis() + 180_000 Exception lastNotFound = null diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy index 5ada1cc5991846..b4a3db04aec64d 100644 --- a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_storage_format_snii", "p0, nonConcurrent") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql "DROP TABLE IF EXISTS test_storage_format_snii" sql "DROP TABLE IF EXISTS test_storage_format_snii_array" sql "DROP TABLE IF EXISTS test_storage_format_snii_add_index" diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_custom_analyzer.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_custom_analyzer.groovy index b0c09e57c99a92..78ce558b7d3017 100644 --- a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_custom_analyzer.groovy +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_custom_analyzer.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_storage_format_snii_custom_analyzer", "p0,nonConcurrent") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def charFilter = "doris_27738_char_filter" def basicAnalyzer = "doris_27738_basic_analyzer" def keywordAnalyzer = "doris_27738_keyword_analyzer" diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_norms.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_norms.groovy index e8888a8d41b78e..697a4116d2a78d 100644 --- a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_norms.groovy +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_norms.groovy @@ -24,6 +24,8 @@ // is refused. // It flips a BE config, so it must not share the cluster with other suites. suite("test_storage_format_snii_norms", "p0,nonConcurrent") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql """ set enable_match_without_inverted_index = false """ sql """ set default_variant_enable_typed_paths_to_sparse = false """ sql """ set default_variant_enable_doc_mode = false """ diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_utf8_wildcard.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_utf8_wildcard.groovy index f069ea44c327c4..b0acf1ab21dbde 100644 --- a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_utf8_wildcard.groovy +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_utf8_wildcard.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_storage_format_snii_utf8_wildcard", "p0, nonConcurrent") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql "DROP TABLE IF EXISTS test_snii_utf8_wildcard" sql "DROP TABLE IF EXISTS test_v3_utf8_wildcard" diff --git a/regression-test/suites/variant_p0/test_variant_search_subcolumn_snii.groovy b/regression-test/suites/variant_p0/test_variant_search_subcolumn_snii.groovy index b69ea4f89cf82d..5d64074e141794 100644 --- a/regression-test/suites/variant_p0/test_variant_search_subcolumn_snii.groovy +++ b/regression-test/suites/variant_p0/test_variant_search_subcolumn_snii.groovy @@ -17,6 +17,8 @@ // Verify SNII SEARCH against direct, nested, and special-character VARIANT paths. suite("test_variant_search_subcolumn_snii") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def table_name = "test_variant_search_subcolumn_snii" sql "set default_variant_doc_materialization_min_rows = 0" diff --git a/regression-test/suites/variant_p0/v2/with_index/test_variant_v2_snii_index.groovy b/regression-test/suites/variant_p0/v2/with_index/test_variant_v2_snii_index.groovy index 9fabd769a92ae4..0e8f7a2511fd76 100644 --- a/regression-test/suites/variant_p0/v2/with_index/test_variant_v2_snii_index.groovy +++ b/regression-test/suites/variant_p0/v2/with_index/test_variant_v2_snii_index.groovy @@ -17,6 +17,8 @@ // Verify SNII indexes written through the VARIANT V2 writer. suite("test_variant_v2_snii_index", "p0,nonConcurrent") { + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" setFeConfigTemporary([enable_variant_v2: true]) { def tblName = "variant_v2_snii_index" def typedInventors = "cast(inventors['inventors'] as array)" diff --git a/regression-test/suites/variant_p0/with_index/test_variant_snii_compaction.groovy b/regression-test/suites/variant_p0/with_index/test_variant_snii_compaction.groovy index 150df9ef69d930..0058fbd0b0250c 100644 --- a/regression-test/suites/variant_p0/with_index/test_variant_snii_compaction.groovy +++ b/regression-test/suites/variant_p0/with_index/test_variant_snii_compaction.groovy @@ -16,6 +16,8 @@ // under the License. suite("regression_test_variant_snii_compaction", "p0, nonConcurrent"){ + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" def table_name = "var_snii_compaction" sql """ set default_variant_enable_typed_paths_to_sparse = false """ sql """ set default_variant_enable_doc_mode = false """ diff --git a/regression-test/suites/variant_p0/with_index/var_index_snii.groovy b/regression-test/suites/variant_p0/with_index/var_index_snii.groovy index cef9d621d2cc49..4c2e79bd8f8112 100644 --- a/regression-test/suites/variant_p0/with_index/var_index_snii.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index_snii.groovy @@ -16,6 +16,8 @@ // under the License. suite("regression_test_variant_var_index_snii", "p0, nonConcurrent"){ + sql "SET enable_common_expr_pushdown=true" + sql "SET enable_common_expr_pushdown_for_inverted_index=true" sql """ set default_variant_enable_typed_paths_to_sparse = false """ sql """ set default_variant_enable_doc_mode = false """ From e9186e438cbb8c13b5490cd8bf464466e6523541 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 1 Oct 2026 15:43:08 +0800 Subject: [PATCH 2/2] [fix](branch-4.2) Keep file cache profile changes out of index CI fix --- be/src/io/cache/block_file_cache_profile.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index a15dd6e5b4c34a..d63ff9cf3a2f1a 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -143,16 +143,6 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(segment_footer_index_bytes_write_into_cache); SUBTRACT_FIELD(remote_only_on_miss_triggered); SUBTRACT_FIELD(remote_only_on_miss_threshold_bytes); - SUBTRACT_FIELD(num_cross_cg_peer_io_total); - SUBTRACT_FIELD(bytes_read_from_cross_cg_peer); - SUBTRACT_FIELD(cross_cg_peer_io_timer); - SUBTRACT_FIELD(num_same_cg_peer_io_total); - SUBTRACT_FIELD(bytes_read_from_same_cg_peer); - SUBTRACT_FIELD(same_cg_peer_io_timer); - SUBTRACT_FIELD(num_peer_race_peer_win); - SUBTRACT_FIELD(num_peer_race_s3_win); - SUBTRACT_FIELD(num_peer_lazy_fetch); - SUBTRACT_FIELD(peer_lazy_fetch_timer); #undef SUBTRACT_FIELD return diff; }