From 756a823beec17096328100fabd4d4e0688a433c0 Mon Sep 17 00:00:00 2001 From: JaySon Date: Mon, 4 Aug 2025 20:09:24 +0800 Subject: [PATCH 1/2] This is an automated cherry-pick of #10325 Signed-off-by: ti-chi-bot --- CMakeLists.txt | 6 +++- dbms/CMakeLists.txt | 11 +++++-- dbms/src/Common/TiFlashBuildInfo.cpp | 5 +++ dbms/src/Common/config.h.in | 1 + .../Coprocessor/DAGStorageInterpreter.cpp | 8 ++++- dbms/src/Server/CMakeLists.txt | 8 +++-- dbms/src/Storages/DeltaMerge/CMakeLists.txt | 6 ++-- .../File/DMFileBlockInputStream.cpp | 19 ++++++++---- .../DeltaMerge/File/DMFileBlockInputStream.h | 11 ++++++- .../DeltaMerge/File/ReadBlockInfo.cpp | 8 ++++- .../DeltaMerge/Filter/PushDownExecutor.cpp | 24 +++++++++++++- .../DeltaMerge/Filter/PushDownExecutor.h | 31 ++++++++++++++++++- .../DeltaMerge/Index/FullTextIndex/Perf.h | 4 +++ .../DeltaMerge/Index/FullTextIndex/Reader.cpp | 5 +++ .../DeltaMerge/Index/FullTextIndex/Reader.h | 4 +++ .../Stream/BruteScoreInputStream.cpp | 4 +++ .../Stream/BruteScoreInputStream.h | 3 ++ .../Stream/ColumnFileInputStream.cpp | 4 +++ .../Stream/ColumnFileInputStream.h | 4 +++ .../Index/FullTextIndex/Stream/Ctx.cpp | 4 +++ .../Index/FullTextIndex/Stream/Ctx.h | 4 +++ .../Stream/DMFileInputStream.cpp | 3 ++ .../FullTextIndex/Stream/DMFileInputStream.h | 4 +++ .../Stream/IProvideFullTextIndex.h | 4 +++ .../FullTextIndex/Stream/InputStream.cpp | 4 +++ .../Index/FullTextIndex/Stream/InputStream.h | 3 ++ .../Stream/ReaderFromColumnFileTiny.cpp | 4 +++ .../Stream/ReaderFromColumnFileTiny.h | 4 +++ .../FullTextIndex/Stream/ReaderFromDMFile.cpp | 3 ++ .../FullTextIndex/Stream/ReaderFromDMFile.h | 4 +++ .../DeltaMerge/Index/FullTextIndex/Writer.cpp | 5 +++ .../DeltaMerge/Index/FullTextIndex/Writer.h | 3 ++ .../tests/gtest_dm_fulltext_index.cpp | 4 +++ .../tests/gtest_dm_fulltext_index_utils.h | 4 +++ .../DeltaMerge/Index/LocalIndexInfo.cpp | 3 ++ .../DeltaMerge/Index/LocalIndexWriter.cpp | 9 +++++- dbms/src/Storages/DeltaMerge/Segment.cpp | 15 ++++++--- dbms/src/Storages/DeltaMerge/Segment.h | 9 +++++- dbms/src/Storages/StorageDeltaMerge.cpp | 15 ++++++++- .../Storages/StorageDisaggregatedRemote.cpp | 5 +++ .../tests/gtests_parse_push_down_filter.cpp | 3 ++ dbms/src/TiDB/Schema/TiDB.cpp | 12 ++++++- dbms/src/TiDB/Schema/TiDB.h | 14 ++++++++- libs/CMakeLists.txt | 2 +- 44 files changed, 280 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71c76218450..b6963eaa676 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,8 @@ include (cmake/utils.cmake) option(ENABLE_PCH "Enable `Precompiled header`" OFF) +option(ENABLE_CLARA "Enable `Clara` library" OFF) + include (cmake/find_ccache.cmake) # Write compile_commands.json @@ -312,7 +314,9 @@ include (cmake/find_tipb.cmake) include (cmake/find_curl.cmake) include (cmake/find_prometheus.cmake) include (cmake/find_tiflash_proxy.cmake) -include (cmake/find_libclara.cmake) +if (ENABLE_CLARA) + include (cmake/find_libclara.cmake) +endif () include (cmake/find_xxhash.cmake) if (OS_LINUX AND ARCH_AMD64 AND TIFLASH_ENABLE_AVX512_SUPPORT) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index e8cc291163e..23cb381e44e 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -224,7 +224,6 @@ target_link_libraries (dbms absl::synchronization tiflash_contrib::aws_s3 tiflash_vector_search - clara_shared etcdpb tiflash_parsers @@ -240,6 +239,10 @@ target_link_libraries (dbms kvstore ) +if (ENABLE_CLARA) + target_link_libraries(dbms clara_shared) +endif () + # always add GmSSL include dir to the include path for static analysis target_include_directories(dbms PRIVATE ${TiFlash_SOURCE_DIR}/contrib/GmSSL/include) if (USE_GM_SSL) @@ -374,7 +377,7 @@ if (ENABLE_TESTS) DESTINATION ".") endif () - if (USE_INTERNAL_LIBCLARA) + if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA) install (TARGETS clara_shared COMPONENT tiflash-gtest DESTINATION ".") @@ -383,7 +386,9 @@ if (ENABLE_TESTS) set_target_properties(gtests_dbms PROPERTIES BUILD_RPATH "$ORIGIN/") set_target_properties(gtests_dbms PROPERTIES INSTALL_RPATH "$ORIGIN/") - install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest) + if (ENABLE_CLARA) + install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest) + endif () target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy) add_check(gtests_dbms) diff --git a/dbms/src/Common/TiFlashBuildInfo.cpp b/dbms/src/Common/TiFlashBuildInfo.cpp index 394b80b3fb7..352051b62f1 100644 --- a/dbms/src/Common/TiFlashBuildInfo.cpp +++ b/dbms/src/Common/TiFlashBuildInfo.cpp @@ -146,6 +146,11 @@ String getEnabledFeatures() #if ENABLE_NEXT_GEN "next-gen", #endif + +// Clara +#if ENABLE_CLARA + "clara", +#endif }; { auto f = DB::DM::VectorIndexHNSWSIMDFeatures::get(); diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 6f72bd1ad5e..5868b45b2eb 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -7,3 +7,4 @@ #cmakedefine01 USE_GM_SSL #cmakedefine01 USE_QPL #cmakedefine01 ENABLE_NEXT_GEN +#cmakedefine01 ENABLE_CLARA diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index a7c42db63a6..ae2cf7a92a3 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -18,6 +18,7 @@ #include #include #include +#include // For ENABLE_CLARA #include #include #include @@ -41,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -62,6 +62,10 @@ #include #include +#if ENABLE_CLARA +#include +#endif + namespace DB { namespace FailPoints @@ -1571,8 +1575,10 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc name = MutSup::extra_table_id_column_name; else if (cid == DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.id) name = DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.name; +#if ENABLE_CLARA else if (cid == DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.id) name = DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.name; +#endif else name = storage_for_logical_table->getTableInfo().getColumnName(cid); required_columns_tmp.emplace_back(std::move(name)); diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 794d05463a9..2efb54cea80 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -124,7 +124,7 @@ if (OS_LINUX) DESTINATION ".") endif() - if (USE_INTERNAL_LIBCLARA) + if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA) install (TARGETS clara_shared COMPONENT tiflash-release DESTINATION ".") @@ -134,12 +134,14 @@ if (OS_LINUX) install (SCRIPT ${TiFlash_SOURCE_DIR}/cmake/tiflash_linux_post_install.cmake COMPONENT tiflash-release) endif() install (SCRIPT ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy-cmake/tiflash_proxy_linux_post_install.cmake COMPONENT tiflash-release) - install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release) + if (ENABLE_CLARA) + install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release) + endif() elseif(APPLE) # set build rpaths, so that executables can be directly called in build tree (easy to debug) set_target_properties(tiflash PROPERTIES BUILD_RPATH "@executable_path/;@executable_path/${TIFLASH_PROXY_LIB_RPATH}/") set_target_properties(tiflash PROPERTIES INSTALL_RPATH "@executable_path/") - if (USE_INTERNAL_LIBCLARA) + if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA) install (TARGETS clara_shared COMPONENT tiflash-release DESTINATION ".") diff --git a/dbms/src/Storages/DeltaMerge/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/CMakeLists.txt index 66e0c7d2d82..70c4d236542 100644 --- a/dbms/src/Storages/DeltaMerge/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/CMakeLists.txt @@ -32,8 +32,10 @@ add_headers_and_sources(delta_merge ./Index/InvertedIndex) add_headers_and_sources(delta_merge ./Index/InvertedIndex/Reader) add_headers_and_sources(delta_merge ./Index/VectorIndex) add_headers_and_sources(delta_merge ./Index/VectorIndex/Stream) -add_headers_and_sources(delta_merge ./Index/FullTextIndex) -add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream) +if (ENABLE_CLARA) + add_headers_and_sources(delta_merge ./Index/FullTextIndex) + add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream) +endif () add_headers_and_sources(delta_merge ./ReadThread) add_headers_and_sources(delta_merge ./Remote) add_headers_and_sources(delta_merge ./Remote/DataStore) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp index 7ecfe509b4d..93efb0fe9a3 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp @@ -12,13 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include // For ENABLE_CLARA #include #include -#include -#include -#include -#include -#include #include #include #include @@ -27,6 +23,13 @@ #include #include +#if ENABLE_CLARA +#include +#include +#include +#include +#include +#endif namespace DB::DM { @@ -145,10 +148,12 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::build( const RowKeyRanges & rowkey_ranges, const ScanContextPtr & scan_context) { + // Note: this file may not have index built { - // Note: this file may not have index built +#if ENABLE_CLARA if (fts_index_ctx) return buildForFullTextIndex(dmfile, read_columns, rowkey_ranges, scan_context); +#endif if (vec_index_ctx) return buildForVectorIndex(dmfile, read_columns, rowkey_ranges, scan_context); } @@ -241,6 +246,7 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForVectorIndex( std::move(rest_columns_reader)); } +#if ENABLE_CLARA SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextIndex( const DMFilePtr & dmfile, const ColumnDefines & read_columns, @@ -327,5 +333,6 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextInde dmfile, std::move(rest_columns_reader)); } +#endif } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h index 8cd831d665f..5ec5f660765 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h @@ -15,18 +15,21 @@ #pragma once #include +#include // For ENABLE_CLARA #include #include #include #include #include #include -#include #include #include #include #include +#if ENABLE_CLARA +#include +#endif namespace DB::DM { @@ -122,11 +125,13 @@ class DMFileBlockInputStreamBuilder return *this; } +#if ENABLE_CLARA DMFileBlockInputStreamBuilder & setFtsIndexQuery(const FullTextIndexStreamCtxPtr & ctx) { fts_index_ctx = ctx; return *this; } +#endif DMFileBlockInputStreamBuilder & setReadPacks(const IdSetPtr & read_packs_) { @@ -201,12 +206,14 @@ class DMFileBlockInputStreamBuilder const RowKeyRanges & rowkey_ranges, const ScanContextPtr & scan_context); +#if ENABLE_CLARA /// The returned stream should be plugged into a FullTextIndexInputStream. Plug to somewhere else will not work. SkippableBlockInputStreamPtr buildForFullTextIndex( const DMFilePtr & dmfile, const ColumnDefines & read_columns, const RowKeyRanges & rowkey_ranges, const ScanContextPtr & scan_context); +#endif private: // These methods are called by the ctor @@ -253,7 +260,9 @@ class DMFileBlockInputStreamBuilder /// If set, will *try* to build a VectorIndexDMFileInputStream /// instead of a normal DMFileBlockInputStream. VectorIndexStreamCtxPtr vec_index_ctx = nullptr; +#if ENABLE_CLARA FullTextIndexStreamCtxPtr fts_index_ctx = nullptr; +#endif // Note: column_cache_long_term is currently only filled when performing Vector Search. ColumnCacheLongTermPtr column_cache_long_term = nullptr; diff --git a/dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp b/dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp index 972fd98809d..6280cc49819 100644 --- a/dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp +++ b/dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include // For ENABLE_CLARA #include -#include #include #include +#if ENABLE_CLARA +#include +#endif + namespace DB::DM { @@ -140,11 +144,13 @@ template ReadBlockInfos ReadBlockInfo::createWithRowIDs( std::span row_ids, const std::vector & pack_offset, const RSResults & pack_res, const DMFileMeta::PackStats & pack_stats, size_t rows_threshold_per_read); +#endif } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp index 4003ebb4810..b2498edc7ab 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include // For ENABLE_CLARA #include #include #include @@ -21,13 +23,16 @@ #include #include #include +#include namespace DB::DM { PushDownExecutorPtr PushDownExecutor::build( const RSOperatorPtr & rs_operator, const ANNQueryInfoPtr & ann_query_info, +#if ENABLE_CLARA const FTSQueryInfoPtr & fts_query_info, +#endif const TiDB::ColumnInfos & table_scan_column_info, const google::protobuf::RepeatedPtrField & pushed_down_filters, const ColumnDefines & columns_to_read, @@ -44,7 +49,13 @@ PushDownExecutorPtr PushDownExecutor::build( if (pushed_down_filters.empty()) { LOG_DEBUG(tracing_logger, "Push down filter is empty"); - return std::make_shared(rs_operator, ann_query_info, fts_query_info, column_range); + return std::make_shared( + rs_operator, + ann_query_info, +#if ENABLE_CLARA + fts_query_info, +#endif + column_range); } std::unordered_map columns_to_read_map; for (const auto & column : columns_to_read) @@ -160,7 +171,9 @@ PushDownExecutorPtr PushDownExecutor::build( return std::make_shared( rs_operator, ann_query_info, +#if ENABLE_CLARA fts_query_info, +#endif before_where, project_after_where, filter_columns, @@ -196,9 +209,14 @@ PushDownExecutorPtr PushDownExecutor::build( ANNQueryInfoPtr ann_query_info = nullptr; if (dag_query->ann_query_info.query_type() != tipb::ANNQueryType::InvalidQueryType) ann_query_info = std::make_shared(dag_query->ann_query_info); +#if ENABLE_CLARA FTSQueryInfoPtr fts_query_info = nullptr; if (dag_query->fts_query_info.query_type() != tipb::FTSQueryType::FTSQueryTypeInvalid) fts_query_info = std::make_shared(dag_query->fts_query_info); +#else + if (dag_query->fts_query_info.query_type() != tipb::FTSQueryType::FTSQueryTypeInvalid) + throw Exception("FTS query is not supported", ErrorCodes::NOT_IMPLEMENTED); +#endif // build push down filter const auto & pushed_down_filters = dag_query->pushed_down_filters; if (unlikely(context.getSettingsRef().force_push_down_all_filters_to_scan) && !dag_query->filters.empty()) @@ -210,7 +228,9 @@ PushDownExecutorPtr PushDownExecutor::build( return PushDownExecutor::build( rs_operator, ann_query_info, +#if ENABLE_CLARA fts_query_info, +#endif columns_to_read_info, merged_filters, columns_to_read, @@ -221,7 +241,9 @@ PushDownExecutorPtr PushDownExecutor::build( return PushDownExecutor::build( rs_operator, ann_query_info, +#if ENABLE_CLARA fts_query_info, +#endif columns_to_read_info, pushed_down_filters, columns_to_read, diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h index 2a324089838..6ecd80b2828 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h @@ -14,12 +14,23 @@ #pragma once +<<<<<<< HEAD +======= +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#pragma GCC diagnostic pop +#include // For ENABLE_CLARA +>>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) #include #include #include -#include #include +#if ENABLE_CLARA +#include +#endif + namespace DB { struct SelectQueryInfo; @@ -38,8 +49,14 @@ class PushDownExecutor PushDownExecutor( const RSOperatorPtr & rs_operator_, const ANNQueryInfoPtr & ann_query_info_, +#if ENABLE_CLARA const FTSQueryInfoPtr & fts_query_info_, +<<<<<<< HEAD const ExpressionActionsPtr & beofre_where_, +======= +#endif + const ExpressionActionsPtr & before_where_, +>>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) const ExpressionActionsPtr & project_after_where_, const ColumnDefinesPtr & filter_columns_, const String filter_column_name_, @@ -54,18 +71,24 @@ class PushDownExecutor , extra_cast(extra_cast_) , columns_after_cast(columns_after_cast_) , ann_query_info(ann_query_info_) +#if ENABLE_CLARA , fts_query_info(fts_query_info_) +#endif , column_range(column_range_) {} explicit PushDownExecutor( const RSOperatorPtr & rs_operator_, const ANNQueryInfoPtr & ann_query_info_ = nullptr, +#if ENABLE_CLARA const FTSQueryInfoPtr & fts_query_info_ = nullptr, +#endif const ColumnRangePtr & column_range_ = nullptr) : rs_operator(rs_operator_) , ann_query_info(ann_query_info_) +#if ENABLE_CLARA , fts_query_info(fts_query_info_) +#endif , column_range(column_range_) {} @@ -73,15 +96,19 @@ class PushDownExecutor : ann_query_info(ann_query_info_) {} +#if ENABLE_CLARA explicit PushDownExecutor(const FTSQueryInfoPtr & fts_query_info_) : fts_query_info(fts_query_info_) {} +#endif // Use by StorageDisaggregated. static PushDownExecutorPtr build( const DM::RSOperatorPtr & rs_operator, const ANNQueryInfoPtr & ann_query_info, +#if ENABLE_CLARA const FTSQueryInfoPtr & fts_query_info, +#endif const TiDB::ColumnInfos & table_scan_column_info, const google::protobuf::RepeatedPtrField & pushed_down_filters, const ColumnDefines & columns_to_read, @@ -116,8 +143,10 @@ class PushDownExecutor const ColumnDefinesPtr columns_after_cast; // The ann_query_info contains the information of the ANN index const ANNQueryInfoPtr ann_query_info; +#if ENABLE_CLARA // The FTSQueryInfo contains the information of the FTS index const FTSQueryInfoPtr fts_query_info; +#endif // The column_range contains the column values of the pushed down filters const ColumnRangePtr column_range; }; diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Perf.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Perf.h index 5eeb52d146d..479982cf55e 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Perf.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Perf.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include @@ -61,3 +64,4 @@ struct FullTextIndexPerf }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.cpp index 06fe0fe3556..cb05f4fc30e 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.cpp @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA + #include #include #include @@ -64,3 +68,4 @@ void FullTextIndexReader::searchScored( return inner->search_scored(query, filter, results); } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.h index abd07640f8d..235a42cdcd3 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Reader.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -60,3 +63,4 @@ class FullTextIndexReader : public ICacheableLocalIndexReader }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.cpp index a94154d839d..61190af6b0b 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -160,3 +163,4 @@ Block FullTextBruteScoreInputStream::getHeader() const } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h index cfce06c7214..327cef972a6 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h @@ -13,7 +13,9 @@ // limitations under the License. #pragma once +#include +#if ENABLE_CLARA #include #include #include @@ -64,3 +66,4 @@ class FullTextBruteScoreInputStream : public NopSkippableBlockInputStream }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.cpp index b2df3bf5d64..ee739f81a8a 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -168,3 +171,4 @@ Block ColumnFileProvideFullTextIndexInputStream::getHeader() const } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.h index 6caa63761d9..60f749fb83a 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ColumnFileInputStream.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -73,3 +76,4 @@ class ColumnFileProvideFullTextIndexInputStream }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.cpp index d0d07c8ed84..d942b858bc7 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -158,3 +161,4 @@ ClaraFTS::BruteScoredSearcher & FullTextIndexStreamCtx::ensureBruteScoredSearche } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h index 4aa884ddf4d..fb1fe15f9c4 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -118,3 +121,4 @@ struct FullTextIndexStreamCtx }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.cpp index 4c4162dbba6..bc2403be568 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.cpp @@ -11,7 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include +#if ENABLE_CLARA #include #include #include @@ -170,3 +172,4 @@ Block DMFileInputStreamProvideFullTextIndex::getHeader() const } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h index 1a0b22a348e..cf5e76881cd 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -76,3 +79,4 @@ class DMFileInputStreamProvideFullTextIndex }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h index b736eec4c0d..c765d91a068 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include @@ -58,3 +61,4 @@ class IProvideFullTextIndex }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.cpp index 6be4483c152..ffaa8fd3d6e 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -207,3 +210,4 @@ FullTextIndexInputStream::FullTextIndexInputStream( } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.h index 43ff8bcdaf1..2ac09255a8d 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/InputStream.h @@ -13,7 +13,9 @@ // limitations under the License. #pragma once +#include +#if ENABLE_CLARA #include #include #include @@ -70,3 +72,4 @@ class FullTextIndexInputStream : public NopSkippableBlockInputStream }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.cpp index b8796b902d4..6be65f48788 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -88,3 +91,4 @@ FullTextIndexReaderPtr FullTextIndexReaderFromColumnFileTiny::load( } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.h index 8c87fbfec0b..c9e0a658614 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromColumnFileTiny.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include @@ -29,3 +32,4 @@ class FullTextIndexReaderFromColumnFileTiny }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.cpp index bdcbb6e2b5b..eb05d24a5ee 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.cpp @@ -11,7 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include +#if ENABLE_CLARA #include #include #include @@ -118,3 +120,4 @@ FullTextIndexReaderPtr FullTextIndexReaderFromDMFile::load( } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.h index 9a92ab2fe50..83ce02ba3d6 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Stream/ReaderFromDMFile.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -28,3 +31,4 @@ class FullTextIndexReaderFromDMFile }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.cpp index b4cd0a12a49..bcb87878c0d 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.cpp @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA + #include #include #include @@ -79,3 +83,4 @@ void FullTextIndexWriterOnDisk::addBlock( } } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.h index de05b2f5067..0c1729380b0 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/Writer.h @@ -13,7 +13,9 @@ // limitations under the License. #pragma once +#include +#if ENABLE_CLARA #include #include #include @@ -101,3 +103,4 @@ class FullTextIndexWriterOnDisk : public LocalIndexWriterOnDisk }; } // namespace DB::DM +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index.cpp b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index.cpp index 49763488c6c..a1d4e321a94 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index.cpp @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#if ENABLE_CLARA #include #include #include @@ -843,3 +846,4 @@ CATCH } // namespace DB::DM::tests +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index_utils.h b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index_utils.h index 9a433557d47..79b2b252f8c 100644 --- a/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index_utils.h +++ b/dbms/src/Storages/DeltaMerge/Index/FullTextIndex/tests/gtest_dm_fulltext_index_utils.h @@ -14,6 +14,9 @@ #pragma once +#include + +#if ENABLE_CLARA #include #include #include @@ -107,3 +110,4 @@ class FullTextIndexTestUtils }; } // namespace DB::DM::tests +#endif diff --git a/dbms/src/Storages/DeltaMerge/Index/LocalIndexInfo.cpp b/dbms/src/Storages/DeltaMerge/Index/LocalIndexInfo.cpp index c37f0040256..2ec7bed775c 100644 --- a/dbms/src/Storages/DeltaMerge/Index/LocalIndexInfo.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/LocalIndexInfo.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include // For ENABLE_CLARA #include #include #include @@ -182,8 +183,10 @@ LocalIndexInfosChangeset generateLocalIndexInfos( // create a new index if (idx.columnarIndexKind() == TiDB::ColumnarIndexKind::Vector) new_index_infos->emplace_back(LocalIndexInfo(idx.id, column_id, idx.vector_index)); +#if ENABLE_CLARA else if (idx.columnarIndexKind() == TiDB::ColumnarIndexKind::FullText) new_index_infos->emplace_back(LocalIndexInfo(idx.id, column_id, idx.full_text_index)); +#endif else if (idx.columnarIndexKind() == TiDB::ColumnarIndexKind::Inverted) new_index_infos->emplace_back(LocalIndexInfo(idx.id, column_id, idx.inverted_index)); newly_added.emplace_back(idx.id); diff --git a/dbms/src/Storages/DeltaMerge/Index/LocalIndexWriter.cpp b/dbms/src/Storages/DeltaMerge/Index/LocalIndexWriter.cpp index 855fb047a61..f8e126b9a59 100644 --- a/dbms/src/Storages/DeltaMerge/Index/LocalIndexWriter.cpp +++ b/dbms/src/Storages/DeltaMerge/Index/LocalIndexWriter.cpp @@ -12,14 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include // For ENABLE_CLARA #include -#include #include #include #include #include #include +#if ENABLE_CLARA +#include +#endif namespace DB::DM { @@ -32,8 +35,10 @@ LocalIndexWriterInMemoryPtr LocalIndexWriter::createInMemory(const LocalIndexInf return std::make_shared(index_info.index_id, index_info.def_vector_index); case TiDB::ColumnarIndexKind::Inverted: return createInMemoryInvertedIndexWriter(index_info.index_id, index_info.def_inverted_index); +#if ENABLE_CLARA case TiDB::ColumnarIndexKind::FullText: return FullTextIndexWriterInMemory::create(index_info.index_id, index_info.def_fulltext_index); +#endif default: RUNTIME_CHECK_MSG(false, "Unsupported index kind: {}", magic_enum::enum_name(index_info.kind)); } @@ -47,8 +52,10 @@ LocalIndexWriterOnDiskPtr LocalIndexWriter::createOnDisk(std::string_view index_ return std::make_shared(index_info.index_id, index_file, index_info.def_vector_index); case TiDB::ColumnarIndexKind::Inverted: return createOnDiskInvertedIndexWriter(index_info.index_id, index_file, index_info.def_inverted_index); +#if ENABLE_CLARA case TiDB::ColumnarIndexKind::FullText: return FullTextIndexWriterOnDisk::create(index_info.index_id, index_info.def_fulltext_index, index_file); +#endif default: RUNTIME_CHECK_MSG(false, "Unsupported index kind: {}", magic_enum::enum_name(index_info.kind)); } diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 72bbb97d23c..d377b038ba3 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -17,6 +17,7 @@ #include #include #include +#include // For ENABLE_CLARA #include #include #include @@ -38,9 +39,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -80,6 +78,11 @@ #include #include +#if ENABLE_CLARA +#include +#include +#include +#endif namespace ProfileEvents { @@ -1301,7 +1304,7 @@ StableValueSpacePtr Segment::prepareMergeDelta( segment_snap, rowkey_range, dm_context.stable_pack_rows, - /*reorginize_block*/ true); + /*reorganize_block*/ true); auto new_stable = createNewStable(dm_context, schema_snap, data_stream, segment_snap->stable->getId(), wbs); @@ -3430,6 +3433,7 @@ BlockInputStreamPtr Segment::getConcatVectorIndexBlockInputStream( return stream3; } +#if ENABLE_CLARA BlockInputStreamPtr Segment::getConcatFullTextIndexBlockInputStream( BitmapFilterPtr bitmap_filter, const SegmentSnapshotPtr & segment_snap, @@ -3491,6 +3495,7 @@ BlockInputStreamPtr Segment::getConcatFullTextIndexBlockInputStream( return stream3; } +#endif BlockInputStreamPtr Segment::getLateMaterializationStream( BitmapFilterPtr & bitmap_filter, @@ -3731,6 +3736,7 @@ BlockInputStreamPtr Segment::getBitmapFilterInputStream( } BlockInputStreamPtr stream; +#if ENABLE_CLARA if (executor && executor->fts_query_info) { return getConcatFullTextIndexBlockInputStream( @@ -3745,6 +3751,7 @@ BlockInputStreamPtr Segment::getBitmapFilterInputStream( read_data_block_rows, ReadTag::Query); } +#endif if (executor && executor->ann_query_info) { // For ANN query, try to use vector index to accelerate. diff --git a/dbms/src/Storages/DeltaMerge/Segment.h b/dbms/src/Storages/DeltaMerge/Segment.h index 8b3615125fe..5c25b2cd6c6 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.h +++ b/dbms/src/Storages/DeltaMerge/Segment.h @@ -14,6 +14,7 @@ #pragma once +#include // For ENABLE_CLARA #include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -32,6 +32,11 @@ #include #include #include + +#if ENABLE_CLARA +#include +#endif + namespace DB { struct GeneralCancelHandle; @@ -776,6 +781,7 @@ class Segment UInt64 start_ts, size_t expected_block_size, ReadTag read_tag); +#if ENABLE_CLARA static BlockInputStreamPtr getConcatFullTextIndexBlockInputStream( BitmapFilterPtr bitmap_filter, const SegmentSnapshotPtr & segment_snap, @@ -787,6 +793,7 @@ class Segment UInt64 start_ts, size_t expected_block_size, ReadTag read_tag); +#endif SkippableBlockInputStreamPtr getConcatSkippableBlockInputStream( const SegmentSnapshotPtr & segment_snap, const DMContext & dm_context, diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 1f974f2d744..c35d61bf02e 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -17,6 +17,7 @@ #include #include #include +#include // For ENABLE_CLARA #include #include #include @@ -44,7 +45,6 @@ #include #include #include -#include #include #include #include @@ -62,6 +62,17 @@ #include #include +<<<<<<< HEAD +======= +#if ENABLE_CLARA +#include +#endif + +namespace CurrentMetrics +{ +extern const Metric DT_NumStorageDeltaMerge; +} // namespace CurrentMetrics +>>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) namespace DB { @@ -686,10 +697,12 @@ void setColumnsToRead( col_define.id = DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.id; col_define.type = DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.type; } +#if ENABLE_CLARA else if (column_names[i] == DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.name) { col_define = DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD; } +#endif else { auto & column = header->getByName(column_names[i]); diff --git a/dbms/src/Storages/StorageDisaggregatedRemote.cpp b/dbms/src/Storages/StorageDisaggregatedRemote.cpp index 34eb8b0f840..8729b1097d3 100644 --- a/dbms/src/Storages/StorageDisaggregatedRemote.cpp +++ b/dbms/src/Storages/StorageDisaggregatedRemote.cpp @@ -16,6 +16,7 @@ #include #include #include +#include // For ENABLE_CLARA #include #include #include @@ -552,14 +553,18 @@ std::variant StorageDisagg DM::ANNQueryInfoPtr ann_query_info = nullptr; if (table_scan.getANNQueryInfo().query_type() != tipb::ANNQueryType::InvalidQueryType) ann_query_info = std::make_shared(table_scan.getANNQueryInfo()); +#if ENABLE_CLARA DM::FTSQueryInfoPtr fts_query_info = nullptr; if (table_scan.getFTSQueryInfo().query_type() != tipb::FTSQueryType::FTSQueryTypeInvalid) fts_query_info = std::make_shared(table_scan.getFTSQueryInfo()); +#endif // build push down executor auto push_down_executor = DM::PushDownExecutor::build( rs_operator, ann_query_info, +#if ENABLE_CLARA fts_query_info, +#endif table_scan.getColumns(), table_scan.getPushedDownFilters(), *column_defines, diff --git a/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp b/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp index e6d3d17d89d..34426b335f3 100644 --- a/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp +++ b/dbms/src/Storages/tests/gtests_parse_push_down_filter.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include // For ENABLE_CLARA #include #include #include @@ -128,7 +129,9 @@ DM::PushDownExecutorPtr generatePushDownExecutor( auto push_down_executor = DM::PushDownExecutor::build( rs_operator, std::make_shared(dag_query->ann_query_info), +#if ENABLE_CLARA std::make_shared(dag_query->fts_query_info), +#endif table_info.columns, pushed_down_filters, columns_to_read, diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index 0ba56e8cbfc..1054e989847 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -15,6 +15,7 @@ #include #include #include +#include // For ENABLE_CLARA #include #include #include @@ -33,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +43,10 @@ #include #include +#if ENABLE_CLARA +#include +#endif + namespace DB { namespace ErrorCodes @@ -129,6 +133,7 @@ enum class IndexType // HNSW = 7, }; +#if ENABLE_CLARA FullTextIndexDefinitionPtr parseFullTextIndexFromJSON(const Poco::JSON::Object::Ptr & json) { RUNTIME_CHECK(json); // not nullptr @@ -154,6 +159,7 @@ Poco::JSON::Object::Ptr fullTextIndexToJSON(const FullTextIndexDefinitionPtr & f json->set("parser_type", full_text_index->parser_type); return json; } +#endif VectorIndexDefinitionPtr parseVectorIndexFromJSON(const Poco::JSON::Object::Ptr & json) { @@ -935,10 +941,12 @@ try { json->set("inverted_index", invertedIndexToJSON(inverted_index)); } +#if ENABLE_CLARA else if (full_text_index) { json->set("full_text_index", fullTextIndexToJSON(full_text_index)); } +#endif #ifndef NDEBUG std::stringstream str; @@ -990,10 +998,12 @@ try { inverted_index = parseInvertedIndexFromJSON(static_cast(index_type), inverted_index_json); } +#if ENABLE_CLARA if (auto full_text_index_json = json->getObject("full_text_index"); full_text_index_json) { full_text_index = parseFullTextIndexFromJSON(full_text_index_json); } +#endif } catch (const Poco::Exception & e) { diff --git a/dbms/src/TiDB/Schema/TiDB.h b/dbms/src/TiDB/Schema/TiDB.h index c2b15f42c49..dae595255c0 100644 --- a/dbms/src/TiDB/Schema/TiDB.h +++ b/dbms/src/TiDB/Schema/TiDB.h @@ -14,6 +14,7 @@ #pragma once +#include // For ENABLE_CLARA #include #include #include @@ -277,7 +278,9 @@ struct IndexInfo VectorIndexDefinitionPtr vector_index = nullptr; InvertedIndexDefinitionPtr inverted_index = nullptr; +#if ENABLE_CLARA FullTextIndexDefinitionPtr full_text_index = nullptr; +#endif ColumnarIndexKind columnarIndexKind() const { @@ -285,12 +288,21 @@ struct IndexInfo return ColumnarIndexKind::Vector; if (inverted_index) return ColumnarIndexKind::Inverted; +#if ENABLE_CLARA if (full_text_index) return ColumnarIndexKind::FullText; +#endif RUNTIME_CHECK(false); } - bool isColumnarIndex() const { return vector_index || inverted_index || full_text_index; } + bool isColumnarIndex() const + { + return vector_index || inverted_index +#if ENABLE_CLARA + || full_text_index +#endif + ; + } }; struct TableInfo diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 3d76ec489f6..6d7917ab99f 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -32,6 +32,6 @@ endif () add_subdirectory (libsymbolization) add_subdirectory (libprocess_metrics) -if (USE_INTERNAL_LIBCLARA) +if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA) add_subdirectory (libclara-cmake) endif () From 9e0072509955c12e0ac0492852769e6f0f06db69 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Mon, 4 Aug 2025 21:04:13 +0800 Subject: [PATCH 2/2] Resolve conflict Signed-off-by: JaySon-Huang --- .../Storages/DeltaMerge/Filter/PushDownExecutor.h | 12 +++--------- dbms/src/Storages/StorageDeltaMerge.cpp | 8 -------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h index 6ecd80b2828..3cc2a188039 100644 --- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h +++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.h @@ -14,14 +14,12 @@ #pragma once -<<<<<<< HEAD -======= #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop + #include // For ENABLE_CLARA ->>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) #include #include #include @@ -51,12 +49,8 @@ class PushDownExecutor const ANNQueryInfoPtr & ann_query_info_, #if ENABLE_CLARA const FTSQueryInfoPtr & fts_query_info_, -<<<<<<< HEAD - const ExpressionActionsPtr & beofre_where_, -======= #endif const ExpressionActionsPtr & before_where_, ->>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) const ExpressionActionsPtr & project_after_where_, const ColumnDefinesPtr & filter_columns_, const String filter_column_name_, @@ -64,7 +58,7 @@ class PushDownExecutor const ColumnDefinesPtr & columns_after_cast_, const ColumnRangePtr & column_range_) : rs_operator(rs_operator_) - , before_where(beofre_where_) + , before_where(before_where_) , project_after_where(project_after_where_) , filter_column_name(std::move(filter_column_name_)) , filter_columns(filter_columns_) @@ -132,7 +126,7 @@ class PushDownExecutor const ExpressionActionsPtr before_where; // The projection after the filter, used to remove the tmp filter column // Used to construct the ExpressionBlockInputStream - // Note: ususally we will remove the tmp filter column in the LateMaterializationBlockInputStream, this only used for unexpected cases + // Note: usually we will remove the tmp filter column in the LateMaterializationBlockInputStream, this only used for unexpected cases const ExpressionActionsPtr project_after_where; const String filter_column_name; // The columns needed by the filter expression diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index c35d61bf02e..ddec9b7874f 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -62,18 +62,10 @@ #include #include -<<<<<<< HEAD -======= #if ENABLE_CLARA #include #endif -namespace CurrentMetrics -{ -extern const Metric DT_NumStorageDeltaMerge; -} // namespace CurrentMetrics ->>>>>>> fc932c9f0d (Storages: Deprecated libclara for normal build (#10325)) - namespace DB { namespace FailPoints