[refactor](be) Unify MATCH and SEARCH execution across inverted index formats

.clang-format-ignore

@@ -8,5 +8,5 @@ be/src/util/sse2neo.h
 be/src/util/sse2neon.h
 be/src/util/mustache/mustache.h
 be/src/util/mustache/mustache.cc
-be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h
+be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h
 cloud/src/common/defer.h

.github/CODEOWNERS

@@ -25,8 +25,7 @@ fe/fe-core/src/main/java/org/apache/doris/fs @CalvinKirs @924060929
 fe/fe-core/src/main/java/org/apache/doris/fsv2 @CalvinKirs
 be/src/olap/rowset/segment_v2/variant @eldenmoon @csun5285
 be/src/storage/index/inverted/ @airborne12 @eldenmoon @csun5285
-be/src/exprs/function/function_search.h @airborne12 @eldenmoon @csun5285
-be/src/exprs/function/function_search.cpp @airborne12 @eldenmoon @csun5285
+be/src/exprs/function/search/ @airborne12 @eldenmoon @csun5285
 be/src/exprs/function/match.cpp @airborne12 @eldenmoon @csun5285
 be/src/exprs/function/match.h @airborne12 @eldenmoon @csun5285
 be/src/exprs/vsearch.h @airborne12 @eldenmoon @csun5285

.licenserc.yaml

@@ -126,7 +126,7 @@ header:
     - "be/src/util/mustache/mustache.h"
     - "be/src/util/sse2neo.h"
     - "be/src/util/sse2neon.h"
-    - "be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h"
+    - "be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h"
     - "be/src/pch/*"
     - "be/test/data"
     - "be/test/expected_result"

LICENSE.txt

@@ -369,7 +369,7 @@ be/src/olap/skiplist.h : BSD-style license
 
 --------------------------------------------------------------------------------
 
-be/src/storage/index/inverted/analyzer/kuromoji/dict/darts.h : 2-Clause BSD license (darts-clone)
+be/src/storage/index/inverted/analysis/analyzer/kuromoji/dict/darts.h : 2-Clause BSD license (darts-clone)
 
   Copyright (c) 2008-2014 Susumu Yata. All rights reserved.
   Vendored from https://github.com/s-yata/darts-clone

NOTICE.txt

@@ -75,7 +75,7 @@ This software includes third party software subject to the following copyrights:
 - Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
 - Darts-clone (double-array trie) - Copyright 2008-2014 Susumu Yata - https://github.com/s-yata/darts-clone (BSD 2-clause)
 - mecab-ipadic (IPADIC) Japanese morphological dictionary - Copyright 2000-2003 Nara Institute of Science and Technology (NAIST) - licensed under NAIST-2003 (BSD-style); the kuromoji analyzer bundles the UTF-8 form from https://github.com/lindera/mecab-ipadic (content of mecab-ipadic-2.7.0-20070801).
-- Apache Lucene - https://github.com/apache/lucene (Apache-2.0): the kuromoji Japanese analyzer under be/src/storage/index/inverted/analyzer/kuromoji is an independent C++ implementation modeled on Lucene's kuromoji analyzer (JapaneseTokenizer), including its search-mode compound-decomposition cost model.
+- Apache Lucene - https://github.com/apache/lucene (Apache-2.0): the kuromoji Japanese analyzer under be/src/storage/index/inverted/analysis/analyzer/kuromoji is an independent C++ implementation modeled on Lucene's kuromoji analyzer (JapaneseTokenizer), including its search-mode compound-decomposition cost model.
 
 The licenses for these third party components are included in LICENSE.txt
 

be/benchmark/benchmark_gram_extractor.hpp

@@ -24,7 +24,7 @@
 #include <string_view>
 #include <vector>
 
-#include "storage/index/inverted/gram/gram_extractor.h"
+#include "storage/index/inverted/analysis/gram/gram_extractor.h"
 
 namespace doris::segment_v2::gram::benchmark_detail {
 

be/src/cloud/cloud_schema_change_job.cpp

@@ -33,7 +33,7 @@
 #include "common/status.h"
 #include "service/backend_options.h"
 #include "storage/delete/delete_handler.h"
-#include "storage/index/inverted/inverted_index_desc.h"
+#include "storage/index/inverted/common/inverted_index_desc.h"
 #include "storage/olap_define.h"
 #include "storage/rowset/beta_rowset.h"
 #include "storage/rowset/rowset.h"

be/src/cloud/cloud_tablet.cpp

@@ -51,7 +51,7 @@
 #include "io/cache/block_file_cache_factory.h"
 #include "storage/compaction/compaction.h"
 #include "storage/compaction/cumulative_compaction_time_series_policy.h"
-#include "storage/index/inverted/inverted_index_desc.h"
+#include "storage/index/inverted/common/inverted_index_desc.h"
 #include "storage/olap_define.h"
 #include "storage/rowset/beta_rowset.h"
 #include "storage/rowset/rowset.h"

be/src/cloud/cloud_warm_up_manager.cpp

@@ -46,7 +46,7 @@
 #include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "service/backend_options.h"
-#include "storage/index/inverted/inverted_index_desc.h"
+#include "storage/index/inverted/common/inverted_index_desc.h"
 #include "storage/rowset/beta_rowset.h"
 #include "storage/tablet/tablet.h"
 #include "util/brpc_client_cache.h" // BrpcClientCache

be/src/exec/operator/olap_scan_operator.cpp

@@ -372,7 +372,6 @@ Status OlapScanLocalState::_init_profile() {
     _index_filter_profile = std::make_unique<RuntimeProfile>("IndexFilter");
     _scanner_profile->add_child(_index_filter_profile.get(), true, nullptr);
     _snii_prx_profile_counters.initialize(_index_filter_profile.get());
-    _snii_phrase_profile_counters.initialize(_index_filter_profile.get());
     /*
     SegmentIterator:
         - AnnIndexLoadCosts: 102.262us

be/src/exec/operator/olap_scan_operator.h

@@ -29,7 +29,7 @@
 #include "exec/operator/scan_operator.h"
 #include "exec/runtime_filter/runtime_filter_bucket_pruner.h"
 #include "runtime/runtime_profile.h"
-#include "storage/index/snii/snii_prx_profile.h"
+#include "storage/index/inverted/snii/snii_prx_profile.h"
 #include "storage/olap_scan_common.h"
 #include "storage/tablet/tablet_reader.h"
 
@@ -160,7 +160,6 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
     std::unique_ptr<RuntimeProfile> _segment_profile;
     std::unique_ptr<RuntimeProfile> _index_filter_profile;
     snii::SniiPrxRuntimeProfileCounters _snii_prx_profile_counters;
-    snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters;
 
     RuntimeProfile::Counter* _condition_cache_lookup_counter = nullptr;
     RuntimeProfile::Counter* _tablet_counter = nullptr;

be/src/exec/scan/olap_scanner.cpp

@@ -42,6 +42,7 @@
 #include "exec/common/variant_util.h"
 #include "exec/operator/olap_scan_operator.h"
 #include "exec/scan/scan_node.h"
+#include "exprs/function/search/search_predicate_collector.h"
 #include "exprs/vexpr.h"
 #include "exprs/vexpr_context.h"
 #include "io/cache/block_file_cache_profile.h"
@@ -54,8 +55,8 @@
 #include "service/backend_options.h"
 #include "storage/binlog.h"
 #include "storage/id_manager.h"
-#include "storage/index/inverted/inverted_index_profile.h"
-#include "storage/index/inverted/similarity/collection_statistics.h"
+#include "storage/index/inverted/common/inverted_index_profile.h"
+#include "storage/index/inverted/reader/collection_statistics.h"
 #include "storage/iterator/block_reader.h"
 #include "storage/olap_common.h"
 #include "storage/olap_tuple.h"
@@ -278,7 +279,8 @@ Status OlapScanner::_prepare_impl() {
 
     if (_tablet_reader_params.score_runtime) {
         SCOPED_TIMER(local_state->_statistics_collect_timer);
-        _tablet_reader_params.collection_statistics = std::make_shared<CollectionStatistics>();
+        _tablet_reader_params.collection_statistics = std::make_shared<CollectionStatistics>(
+                std::make_unique<SearchPredicateCollector>());
 
         auto io_ctx = build_score_runtime_collection_io_context(
                 _state, _tablet_reader_params.reader_type, tablet->file_cache_ttl_expiration_time(),
@@ -884,7 +886,6 @@ void OlapScanner::_collect_profile_before_close() {
                    stats.inverted_index_analyzer_timer);
     COUNTER_UPDATE(local_state->_inverted_index_lookup_timer, stats.inverted_index_lookup_timer);
     local_state->_snii_prx_profile_counters.update(stats);
-    local_state->_snii_phrase_profile_counters.update(stats);
     COUNTER_UPDATE(local_state->_variant_scan_sparse_column_timer,
                    stats.variant_scan_sparse_column_timer_ns);
     COUNTER_UPDATE(local_state->_variant_scan_sparse_column_bytes,

be/src/exprs/function/array/function_array_index.h

@@ -49,7 +49,7 @@
 #include "exprs/expr_zonemap_filter.h"
 #include "exprs/function/function.h"
 #include "storage/index/index_reader_helper.h"
-#include "storage/index/inverted/inverted_index_query_type.h"
+#include "storage/index/inverted/common/inverted_index_query_type.h"
 #include "storage/index/inverted/inverted_index_reader.h"
 #include "storage/predicate/column_predicate.h"
 

be/src/exprs/function/function_multi_match.cpp

@@ -29,7 +29,6 @@
 #include "exprs/function/simple_function_factory.h"
 #include "exprs/vslot_ref.h"
 #include "io/fs/file_reader.h"
-#include "storage/index/inverted/query/phrase_prefix_query.h"
 #include "storage/segment/segment_iterator.h"
 
 namespace doris {