diff --git a/be/src/core/CMakeLists.txt b/be/src/core/CMakeLists.txt index 31e12dc4b5ca18..7459eca6b6749b 100644 --- a/be/src/core/CMakeLists.txt +++ b/be/src/core/CMakeLists.txt @@ -23,6 +23,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/core") file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) add_library(Core STATIC ${SRC_FILES}) +target_link_libraries(Core PUBLIC bson2) pch_reuse(Core) diff --git a/be/src/core/data_type_serde/bson_value_view.cpp b/be/src/core/data_type_serde/bson_value_view.cpp new file mode 100644 index 00000000000000..2714ff6ea005e5 --- /dev/null +++ b/be/src/core/data_type_serde/bson_value_view.cpp @@ -0,0 +1,328 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/bson_value_view.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace doris { + +const BsonBinding* BsonBinding::child(const std::string& child_name, size_t ordinal) const { + if (ordinal < children.size() && children[ordinal].name == child_name) { + return &children[ordinal]; + } + auto position = child_positions.find(child_name); + if (position != child_positions.end()) { + return &children[position->second]; + } + // Programmatically constructed bindings (including tests) may omit the derived index. + if (child_positions.empty()) { + for (const auto& candidate : children) { + if (candidate.name == child_name) { + return &candidate; + } + } + } + return nullptr; +} + +BsonValueView BsonValueView::of(const bson_value_t& v) { + BsonValueView result; + result.value = v; + result.missing = false; + return result; +} + +BsonValueView BsonValueView::document(const bson_t& document) { + bson_value_t value {}; + value.value_type = BSON_TYPE_DOCUMENT; + value.value.v_doc.data = const_cast(bson_get_data(&document)); + value.value.v_doc.data_len = document.len; + return of(value); +} + +BsonValueView BsonValueView::at_path(const std::vector& path) const { + BsonValueView current = *this; + for (const auto& segment : path) { + if (current.is_null()) { + return {}; + } + if (current.value.value_type != BSON_TYPE_DOCUMENT) { + // For path [a,b], {a:1} is a type mismatch; {a:{}} is a genuinely missing child. + BsonValueView mismatch; + mismatch.path_mismatch = true; + return mismatch; + } + bson_t document; + bson_iter_t it; + if (!bson_init_static(&document, current.value.value.v_doc.data, + current.value.value.v_doc.data_len) || + !bson_iter_init_find(&it, &document, segment.c_str())) { + return {}; + } + current = of(*bson_iter_value(&it)); + } + return current; +} + +BsonDecodeContext BsonDecodeContext::child(const BsonBinding* child_binding) const { + return {.binding = child_binding, .mismatch_as_null = mismatch_as_null, .depth = depth + 1}; +} + +Status BsonDecodeContext::validate(const BsonValueView& value) const { + if (value.path_mismatch) { + return Status::InvalidArgument("BSON field path traverses a non-document value"); + } + if (depth > MAX_DEPTH) { + return Status::InvalidArgument("MongoDB BSON nesting exceeds the decoding limit"); + } + if (value.is_null()) { + return Status::OK(); + } + if (binding && !binding->types.empty() && + std::ranges::find(binding->types, static_cast(value.value.value_type)) == + binding->types.end()) { + return Status::InvalidArgument("MongoDB BSON type does not match the pinned schema"); + } + if ((value.value.value_type == BSON_TYPE_DOCUMENT || + value.value.value_type == BSON_TYPE_ARRAY) && + value.value.value.v_doc.data_len > MAX_DOCUMENT_BYTES) { + return Status::InvalidArgument("MongoDB BSON document exceeds the decoding size limit"); + } + return Status::OK(); +} + +namespace { +Status parse_binding(const rapidjson::Value& value, BsonBinding* out, int depth, size_t* count) { + if (!value.IsObject() || depth > BsonDecodeContext::MAX_DEPTH || ++*count > 4096 || + !value.HasMember("name") || !value["name"].IsString() || !value.HasMember("path") || + !value["path"].IsArray() || !value.HasMember("bson_types") || + !value["bson_types"].IsArray() || !value.HasMember("children") || + !value["children"].IsArray()) { + return Status::InvalidArgument("Invalid MongoDB binding descriptor"); + } + out->name = value["name"].GetString(); + for (const auto& segment : value["path"].GetArray()) { + if (!segment.IsString() || std::memchr(segment.GetString(), 0, segment.GetStringLength())) { + return Status::InvalidArgument("Invalid MongoDB binding path"); + } + out->path.emplace_back(segment.GetString(), segment.GetStringLength()); + } + for (const auto& type : value["bson_types"].GetArray()) { + if (!type.IsInt()) { + return Status::InvalidArgument("Invalid MongoDB BSON tag"); + } + out->types.push_back(type.GetInt()); + } + if (value.HasMember("subtype") && value["subtype"].IsInt()) { + out->subtype = value["subtype"].GetInt(); + } + if (value.HasMember("special") && value["special"].IsString()) { + out->special = value["special"].GetString(); + } + for (const auto& child : value["children"].GetArray()) { + BsonBinding binding; + RETURN_IF_ERROR(parse_binding(child, &binding, depth + 1, count)); + if (!out->child_positions.emplace(binding.name, out->children.size()).second) { + return Status::InvalidArgument("Duplicate BSON child binding"); + } + out->children.push_back(std::move(binding)); + } + return Status::OK(); +} +} // namespace + +Status parse_bson_bindings(const std::string& json, std::vector* bindings) { + if (json.size() > 1024 * 1024) { + return Status::InvalidArgument("MongoDB schema descriptor exceeds the size limit"); + } + rapidjson::Document document; + document.Parse(json.data(), json.size()); + if (document.HasParseError() || !document.IsObject() || !document.HasMember("columns") || + !document["columns"].IsArray() || document["columns"].Empty() || + !document.HasMember("version") || !document["version"].IsInt() || + document["version"].GetInt() != 1) { + return Status::InvalidArgument("Invalid MongoDB schema descriptor"); + } + size_t count = 0; + std::vector result; + for (const auto& column : document["columns"].GetArray()) { + BsonBinding binding; + RETURN_IF_ERROR(parse_binding(column, &binding, 0, &count)); + if (std::ranges::any_of(result, [&](const auto& b) { return b.name == binding.name; })) { + return Status::InvalidArgument("Duplicate MongoDB binding name"); + } + result.push_back(std::move(binding)); + } + *bindings = std::move(result); + return Status::OK(); +} + +Status bson_decimal_exact(const bson_decimal128_t& value, int precision, int scale, + std::string* text) { + char raw[BSON_DECIMAL128_STRING]; + bson_decimal128_to_string(&value, raw); + std::string source(raw); + size_t pos = source[0] == '-' || source[0] == '+' ? 1 : 0; + bool negative = source[0] == '-'; + std::string digits; + int fractional = 0; + bool point = false; + for (; pos < source.size() && source[pos] != 'E' && source[pos] != 'e'; ++pos) { + char c = source[pos]; + if (c == '.' && !point) { + point = true; + } else if (c >= '0' && c <= '9') { + digits += c; + fractional += point; + } else { + return Status::InvalidArgument( + "Non-finite BSON Decimal128 has no Doris DECIMAL representation"); + } + } + int exponent = 0; + if (pos < source.size()) { + ++pos; + bool plus = source[pos] == '+'; + pos += plus; + auto parsed = std::from_chars(source.data() + pos, source.data() + source.size(), exponent); + if (parsed.ec != std::errc() || parsed.ptr != source.data() + source.size()) { + return Status::InvalidArgument("Invalid BSON Decimal128 exponent"); + } + } + auto first = digits.find_first_not_of('0'); + if (first == std::string::npos) { + *text = "0"; + return Status::OK(); + } + digits.erase(0, first); + int shift = scale + exponent - fractional; + if (shift < 0) { + auto removed = static_cast(-shift); + if (removed >= digits.size() || digits.find_last_not_of('0') >= digits.size() - removed) { + return Status::InvalidArgument("BSON Decimal128 requires rounding at the target scale"); + } + digits.resize(digits.size() - removed); + } else { + if (digits.size() + shift > static_cast(precision)) { + return Status::InvalidArgument("BSON Decimal128 exceeds the target precision"); + } + digits.append(shift, '0'); + } + if (digits.size() > static_cast(precision)) { + return Status::InvalidArgument("BSON Decimal128 exceeds the target precision"); + } + if (scale > 0) { + if (digits.size() <= static_cast(scale)) { + digits.insert(0, scale + 1 - digits.size(), '0'); + } + digits.insert(digits.size() - scale, 1, '.'); + } + *text = (negative ? "-" : "") + digits; + return Status::OK(); +} + +Status validate_bson_tree(const BsonValueView& root) { + auto is_container = [](bson_type_t type) { + return type == BSON_TYPE_DOCUMENT || type == BSON_TYPE_ARRAY || + type == BSON_TYPE_CODEWSCOPE; + }; + // Scalars use another union member and expose no child-document bytes to traverse. + if (!is_container(root.value.value_type)) { + return Status::OK(); + } + // A wide array of empty documents can fit in 16 MiB yet queue tens of MiB of sibling + // values. Retain only one iterator per ancestor: scratch is bounded by depth, not width. + std::array pending; + auto enter = [&](const bson_value_t& value, int depth) -> Status { + // The root occupies depth zero; reject the next nested container before indexing the stack. + if (depth > BsonDecodeContext::MAX_DEPTH) { + return Status::InvalidArgument("BSON document exceeds the nesting limit"); + } + bool scope = value.value_type == BSON_TYPE_CODEWSCOPE; + const uint8_t* data = scope ? value.value.v_codewscope.scope_data : value.value.v_doc.data; + uint32_t length = scope ? value.value.v_codewscope.scope_len : value.value.v_doc.data_len; + bson_t document; + if (length > BsonDecodeContext::MAX_DOCUMENT_BYTES || + !bson_init_static(&document, data, length) || + !bson_iter_init(&pending[depth], &document)) { + return Status::Corruption("Invalid nested BSON document"); + } + return Status::OK(); + }; + int depth = 0; + RETURN_IF_ERROR(enter(root.value, depth)); + size_t elements = 0; + while (depth >= 0) { + // Resume the parent after this container ends instead of queueing all its siblings. + if (!bson_iter_next(&pending[depth])) { + --depth; + continue; + } + if (++elements > BsonDecodeContext::MAX_ELEMENTS) { + return Status::InvalidArgument("BSON document exceeds the element limit"); + } + const auto* value = bson_iter_value(&pending[depth]); + if (is_container(value->value_type)) { + RETURN_IF_ERROR(enter(*value, ++depth)); + } + } + return Status::OK(); +} + +Status bson_canonical_json(const BsonValueView& value, std::string* json) { + if (value.missing) { + return Status::InvalidArgument("Missing BSON value cannot be materialized as JSON"); + } + RETURN_IF_ERROR(validate_bson_tree(value)); + bson_t wrapper; + bson_init(&wrapper); + if (!bson_append_value(&wrapper, "v", 1, &value.value)) { + bson_destroy(&wrapper); + return Status::InvalidArgument( + "BSON value cannot be represented as canonical Extended JSON"); + } + size_t length = 0; + // Copying the result can throw on memory limits; keep the driver buffer owned during that copy. + std::unique_ptr encoded( + bson_as_canonical_extended_json(&wrapper, &length), &bson_free); + bson_destroy(&wrapper); + if (!encoded) { + return Status::InvalidArgument("BSON canonical Extended JSON encoding failed"); + } + // libbson emits { "v" : VALUE }. The wrapper is fixed; VALUE may itself contain braces or quotes. + std::string_view result(encoded.get(), length); + size_t begin = result.find(':') + 1; + size_t end = result.find_last_of('}'); + while (begin < end && result[begin] == ' ') { + ++begin; + } + while (end > begin && result[end - 1] == ' ') { + --end; + } + json->assign(result.substr(begin, end - begin)); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/core/data_type_serde/bson_value_view.h b/be/src/core/data_type_serde/bson_value_view.h new file mode 100644 index 00000000000000..3fa37e06f477c0 --- /dev/null +++ b/be/src/core/data_type_serde/bson_value_view.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris { + +// Type tags and exact path segments are pinned by FE, independently of the projected Doris type. +struct BsonBinding { + // Doris names are lowercase; source path segments retain their exact case and punctuation. + std::string name; + std::vector path; + // An empty tag list leaves physical-type validation to the target SerDe. + std::vector types; + std::vector children; + // BinData-to-VARBINARY requires an explicit subtype (0..255); -1 means unspecified. + int subtype = -1; + // Empty selects path lookup; "ejson" and "bson" materialize the entire source document. + std::string special; + // Derived name index built while parsing; it is not part of the serialized FE contract. + std::unordered_map child_positions; + + // Resolve a projected child by name, using ordinal only when its name still matches. For + // STRUCT projected as STRUCT, ordinal zero must resolve b rather than the original a. + // The returned pointer borrows this binding's children and is null when the name is absent. + const BsonBinding* child(const std::string& child_name, size_t ordinal) const; +}; + +// A shallow view of one BSON value. Payload pointers borrow the source document; SerDes must copy +// bytes into Doris columns before the cursor advances or closes. This view never frees BSON data. +struct BsonValueView { + bson_value_t value {}; + bool missing = true; + bool path_mismatch = false; + + // Mark a BSON value as present, copying its scalar/tag fields and borrowing any payload pointers. + static BsonValueView of(const bson_value_t& value); + + // View the document's original bytes as BSON_TYPE_DOCUMENT without copying or taking ownership. + static BsonValueView document(const bson_t& document); + + // Traverse case-sensitive document keys relative to this view; ["a.b"] and ["a","b"] differ. + // An empty path returns this view. Missing/null ancestors yield missing; a scalar ancestor sets + // path_mismatch, so {a:1} at [a,b] cannot silently become the same SQL NULL as {a:{}}. + BsonValueView at_path(const std::vector& path) const; + + // Missing and BSON null both map to SQL NULL. A failed scalar traversal remains a type error. + bool is_null() const { + return !path_mismatch && (missing || value.value_type == BSON_TYPE_NULL); + } +}; + +// Per-value decoding policy. Bindings are owned by the reader; child contexts borrow the same tree. +struct BsonDecodeContext { + const BsonBinding* binding = nullptr; + // Nullable SerDe applies this only to a top-level field (depth zero), after nested rollback. + bool mismatch_as_null = false; + int depth = 0; + static constexpr int MAX_DEPTH = 64; + static constexpr size_t MAX_DOCUMENT_BYTES = 16 * 1024 * 1024; + static constexpr size_t MAX_ELEMENTS = 1000000; + + // Carry the mismatch policy into a child while replacing its binding and increasing depth. + BsonDecodeContext child(const BsonBinding* child_binding) const; + + // Check path traversal, depth, declared BSON tags and container size before typed conversion. + // Null/missing are permitted here; the target SerDe handles nullability, ranges and subtypes. + Status validate(const BsonValueView& value) const; +}; + +// Parse FE's version-1 bindings JSON, enforcing 1 MiB, 4096-field and nesting limits and rejecting +// duplicate names. Replace bindings only on success; returned bindings own all strings and children. +Status parse_bson_bindings(const std::string& json, std::vector* bindings); + +// Format a finite Decimal128 exactly for DECIMAL(precision,scale), without rounding. For scale 2, +// 1.2300 becomes 1.23; 1.234, non-finite values and precision overflow return an error. +Status bson_decimal_exact(const bson_decimal128_t& value, int precision, int scale, + std::string* text); + +// Check document/array/code-with-scope subtrees depth-first with fixed, depth-bounded scratch +// against depth, size and element limits before conversion; this does not validate a target SQL type. +Status validate_bson_tree(const BsonValueView& value); + +// Encode one present value as owned canonical Extended JSON, retaining BSON tags such as $oid and +// $numberLong. Reject missing or over-limit trees; Nullable SerDe handles SQL NULL before this call. +Status bson_canonical_json(const BsonValueView& value, std::string* json); + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index 268bb48ef0521c..8c1c2f644f684c 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -33,11 +33,13 @@ #include "core/data_type/get_least_supertype.h" #include "core/data_type_serde/arrow_parent_nulls.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/complex_type_deserialize_util.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "exprs/function/function_helpers.h" #include "storage/field_type.h" +#include "util/defer_op.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" @@ -802,4 +804,47 @@ Status DataTypeArraySerDe::read_column_from_orc(IColumn& column, return decode_list_orc_values(nested_serde, column, view); } +Status DataTypeArraySerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing || value.value.value_type != BSON_TYPE_ARRAY || !context.binding || + context.binding->children.size() != 1) { + return Status::InvalidArgument("BSON ARRAY requires one pinned element binding"); + } + bson_t document; + bson_iter_t it; + if (!bson_init_static(&document, value.value.value.v_doc.data, + value.value.value.v_doc.data_len) || + !bson_iter_init(&it, &document)) { + return Status::InvalidArgument("Invalid BSON array"); + } + auto& target = assert_cast(column); + auto& data = target.get_data(); + const size_t before = data.size(); + const size_t offsets_before = target.get_offsets().size(); + bool committed = false; + Defer rollback([&] { + if (!committed) { + if (data.size() > before) { + data.pop_back(data.size() - before); + } + target.get_offsets().resize(offsets_before); + } + }); + size_t count = 0; + while (bson_iter_next(&it)) { + Status status = ++count > BsonDecodeContext::MAX_ELEMENTS + ? Status::InvalidArgument("BSON array exceeds the element limit") + : nested_serde->read_one_cell_from_bson( + data, BsonValueView::of(*bson_iter_value(&it)), + context.child(context.binding->children.data())); + if (!status.ok()) { + return status; + } + } + target.get_offsets().push_back(data.size()); + committed = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_array_serde.h b/be/src/core/data_type_serde/data_type_array_serde.h index 31378a61081ede..3d55eccc856d91 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.h +++ b/be/src/core/data_type_serde/data_type_array_serde.h @@ -46,6 +46,9 @@ class DataTypeArraySerDe : public DataTypeSerDe { nested_serde = std::move(nullable_serde); } + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return "Array(" + nested_serde->get_name() + ")"; } Status from_string(StringRef& str, IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 9df0a8e060a22b..19dcd3b3344a07 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/orc_timestamp_decode_context.h" @@ -1076,4 +1077,43 @@ Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, return decode_timestamp_orc_values(column, view); } +Status DataTypeDateTimeV2SerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing || value.value.value_type != BSON_TYPE_DATE_TIME) { + return Status::InvalidArgument("BSON DATETIME requires a date source"); + } + int64_t millis = value.value.value.v_datetime; + int64_t seconds = millis / 1000; + int64_t remainder = millis % 1000; + if (remainder < 0) { + --seconds; + remainder += 1000; + } + int64_t quantum = 1; + for (int i = _scale; i < 3; ++i) { + quantum *= 10; + } + if (remainder % quantum != 0) { + return Status::InvalidArgument("BSON date requires rounding at the target datetime scale"); + } + const auto civil = cctz::convert(cctz::time_point(cctz::seconds(seconds)), + cctz::utc_time_zone()); + if (civil.year() < 0 || civil.year() > 9999) { + return Status::InvalidArgument("BSON date is outside the DATETIMEV2 range"); + } + DateV2Value result; + result.unchecked_set_time( + static_cast(civil.year()), static_cast(civil.month()), + static_cast(civil.day()), static_cast(civil.hour()), + static_cast(civil.minute()), static_cast(civil.second()), + static_cast(remainder * 1000)); + if (!result.is_valid_date()) { + return Status::InvalidArgument("BSON date is not representable as DATETIMEV2"); + } + assert_cast(column).insert_value( + binary_cast(result)); + return Status::OK(); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index 013e37b7361e3a..48c283d6f4d33c 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -35,6 +35,9 @@ class Arena; class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe { public: + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + DataTypeDateTimeV2SerDe(int scale, int nesting_level = 1) : DataTypeNumberSerDe(nesting_level), _scale(scale) {}; diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index c43efb709ecf5f..b38a4eba77f463 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -36,6 +36,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" @@ -1505,6 +1506,21 @@ Status DataTypeDecimalSerDe::read_column_from_orc(IColumn& column, return decode_decimal_orc_values(*this, column, view, cast_set(scale)); } +template +Status DataTypeDecimalSerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing || value.value.value_type != BSON_TYPE_DECIMAL128) { + return Status::InvalidArgument("BSON DECIMAL requires a Decimal128 source"); + } + std::string normalized; + RETURN_IF_ERROR( + bson_decimal_exact(value.value.value.v_decimal128, precision, scale, &normalized)); + StringRef input(normalized); + Status status = from_string_strict_mode(input, column, FormatOptions()); + return status.ok() ? status : Status::InvalidArgument("BSON Decimal128 conversion failed"); +} + template class DataTypeDecimalSerDe; template class DataTypeDecimalSerDe; template class DataTypeDecimalSerDe; diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index e9f7db67e0b820..f1887cd536c106 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -51,6 +51,9 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { scale_multiplier(decimal_scale_multiplier(scale)), _parquet_predicate_values(0, static_cast(scale)) {} + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return type_to_string(T); } Status from_string_batch(const ColumnString& str, ColumnNullable& column, diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp index bf9252342e9856..b7e79184477f71 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp @@ -26,6 +26,7 @@ #include "common/exception.h" #include "common/status.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/value/jsonb_value.h" #include "exprs/json_functions.h" #include "util/jsonb_parser_simd.h" @@ -378,4 +379,15 @@ bool DataTypeJsonbSerDe::write_column_to_presto_text(const IColumn& column, Buff return true; } +Status DataTypeJsonbSerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + std::string json; + RETURN_IF_ERROR(bson_canonical_json(value, &json)); + Slice slice(json); + Status status = deserialize_one_cell_from_json(column, slice, FormatOptions()); + return status.ok() ? status + : Status::InvalidArgument("BSON Extended JSON exceeds JSONB limits"); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.h b/be/src/core/data_type_serde/data_type_jsonb_serde.h index 3a1e5b6df77798..8c4d3c55de2ba3 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.h +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.h @@ -35,6 +35,9 @@ class DataTypeJsonbSerDe : public DataTypeStringSerDe { public: DataTypeJsonbSerDe(int nesting_level = 1) : DataTypeStringSerDe(TYPE_JSONB, nesting_level) {}; + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return "JSONB"; } // JSON null is a value, so a nested SQL NULL needs a distinct spelling. diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index 6f191db7a81bc3..90fe0af9fb3c5c 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -33,12 +33,14 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" #include "exprs/function/cast/cast_base.h" #include "format/transformer/vcsv_transformer.h" +#include "util/defer_op.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" #include "util/simd/bits.h" @@ -695,4 +697,40 @@ Status DataTypeNullableSerDe::read_column_from_orc(IColumn& column, } return st; } +Status DataTypeNullableSerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + auto& target = assert_cast(column); + const size_t before = target.size(); + bool committed = false; + Defer rollback([&] { + if (!committed) { + auto& nested = target.get_nested_column(); + if (nested.size() > before) { + nested.pop_back(nested.size() - before); + } + target.get_null_map_data().resize(before); + } + }); + if (value.is_null()) { + target.insert_default(); + committed = true; + return Status::OK(); + } + Status status = + nested_serde->read_one_cell_from_bson(target.get_nested_column(), value, context); + if (!status.ok()) { + if (context.mismatch_as_null && context.depth == 0 && + status.code() == ErrorCode::INVALID_ARGUMENT) { + // Nested SerDes roll back first: [1, "bad"] must yield one NULL, not a stray array element. + target.insert_default(); + committed = true; + return Status::OK(); + } + return status; + } + target.get_null_map_data().push_back(0); + committed = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index e43da468806b70..119e68f4fd9c36 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -34,6 +34,9 @@ class DataTypeNullableSerDe : public DataTypeSerDe { DataTypeNullableSerDe(const DataTypeSerDeSPtr& _nested_serde, int nesting_level = 1) : DataTypeSerDe(nesting_level), nested_serde(_nested_serde) {} + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return "Nullable(" + nested_serde->get_name() + ")"; } Status from_string_as_nested(StringRef& str, IColumn& column, const FormatOptions& options, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 9eab9b00dbb11d..4d640085f7839e 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -37,6 +37,7 @@ #include "core/data_type/primitive_type.h" #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" @@ -2176,6 +2177,53 @@ Status DataTypeNumberSerDe::write_column_to_pb(const IColumn& column, PValues } /// Explicit template instantiations - to avoid code bloat in headers. +template +Status DataTypeNumberSerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + const auto& v = value.value; + if (value.missing) { + return Status::InvalidArgument("Missing BSON value requires a nullable target"); + } + if constexpr (T == TYPE_BOOLEAN) { + if (v.value_type != BSON_TYPE_BOOL) { + return Status::InvalidArgument("BSON boolean requires a boolean source"); + } + assert_cast(column).insert_value(v.value.v_bool); + } else if constexpr (is_int(T)) { + if constexpr (T == TYPE_BIGINT) { + if (v.value_type == BSON_TYPE_DATE_TIME) { + assert_cast(column).insert_value(v.value.v_datetime); + return Status::OK(); + } + } + if (v.value_type != BSON_TYPE_INT32 && v.value_type != BSON_TYPE_INT64) { + return Status::InvalidArgument("BSON integer requires an integer source"); + } + int64_t n = v.value_type == BSON_TYPE_INT32 ? v.value.v_int32 : v.value.v_int64; + using Native = typename PrimitiveTypeTraits::CppType; + if (n < std::numeric_limits::lowest() || n > std::numeric_limits::max()) { + return Status::InvalidArgument("BSON integer exceeds the target range"); + } + assert_cast(column).insert_value(static_cast(n)); + } else if constexpr (T == TYPE_DOUBLE || T == TYPE_FLOAT) { + if (v.value_type != BSON_TYPE_DOUBLE) { + return Status::InvalidArgument("BSON floating point requires a double source"); + } + double n = v.value.v_double; + if constexpr (T == TYPE_FLOAT) { + if (std::isfinite(n) && static_cast(static_cast(n)) != n) { + return Status::InvalidArgument( + "BSON double cannot be represented exactly as FLOAT"); + } + } + assert_cast(column).insert_value(n); + } else { + return Status::NotSupported("BSON decoding is not supported for this numeric type"); + } + return Status::OK(); +} + template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 47674c48cc5750..c2be240e82465f 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -57,6 +57,9 @@ class DataTypeNumberSerDe : public DataTypeSerDe { DataTypeNumberSerDe(int nesting_level = 1) : DataTypeSerDe(nesting_level) {}; + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return type_to_string(T); } Status from_string(StringRef& str, IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index ab95bd7f0bece7..40f0c857864aaf 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -16,6 +16,7 @@ // under the License. #include "core/data_type_serde/data_type_serde.h" +#include #include #include "common/check.h" @@ -26,6 +27,7 @@ #include "core/column/column_nullable.h" #include "core/data_type/data_type.h" #include "core/data_type/storage_field_type.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_decimal_serde.h" #include "core/data_type_serde/data_type_jsonb_serde.h" @@ -399,4 +401,26 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_field(const uint8_t* data, F return end; } +Status DataTypeSerDe::read_one_cell_from_bson(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + const size_t before = column.size(); + Status status; + try { + status = decode_bson_value(column, value, context); + } catch (const Exception& error) { + status = error.to_status(); + } catch (const std::bad_alloc&) { + status = Status::MemoryLimitExceeded("BSON decoding allocation failed"); + } + if (!status.ok() && column.size() > before) { + column.pop_back(column.size() - before); + } + return status; +} + +Status DataTypeSerDe::decode_bson_value(IColumn&, const BsonValueView&, + const BsonDecodeContext&) const { + return Status::NotSupported("BSON decoding is not supported for this Doris type"); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 7618322300eacf..0693d68b731896 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -91,6 +91,8 @@ class Arena; class IDataType; struct CastParameters; +struct BsonValueView; +struct BsonDecodeContext; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; @@ -493,6 +495,11 @@ class DataTypeSerDe { virtual void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const = 0; + Status read_one_cell_from_bson(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const; + virtual Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const; + // return true if output as string // return false if output null virtual bool write_column_to_mysql_text(const IColumn& column, BufferWritable& bw, diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 315e4ed4cbe7b6..5096b4ef704387 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -30,6 +30,7 @@ #include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" @@ -931,6 +932,21 @@ Status DataTypeStringSerDeBase::read_column_from_orc( return decode_string_orc_values(*this, column, view); } +template +Status DataTypeStringSerDeBase::decode_bson_value( + IColumn& column, const BsonValueView& value, const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing || value.value.value_type != BSON_TYPE_UTF8) { + return Status::InvalidArgument("BSON STRING requires a UTF-8 string source"); + } + const auto& v = value.value.value.v_utf8; + if (!bson_utf8_validate(v.str, v.len, true)) { + return Status::InvalidArgument("BSON string contains invalid UTF-8"); + } + assert_cast(column).insert_data(v.str, v.len); + return Status::OK(); +} + template class DataTypeStringSerDeBase; template class DataTypeStringSerDeBase; template class DataTypeStringSerDeBase; diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index df65f438fe7ebe..80bf3f47123e1e 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -97,6 +97,9 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { DataTypeStringSerDeBase(PrimitiveType type, int nesting_level = 1, int len = -1) : DataTypeSerDe(nesting_level), _type(type), _len(len) {} + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override { return "String"; } Status from_string(StringRef& str, IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index 7dbf40ef7bd925..28d5b351160573 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -31,10 +31,12 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/arrow_parent_nulls.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/complex_type_deserialize_util.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" +#include "util/defer_op.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" @@ -820,4 +822,78 @@ Status DataTypeStructSerDe::read_column_from_orc(IColumn& column, return decode_struct_orc_values(elem_serdes_ptrs, column, view); } +Status DataTypeStructSerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing || !context.binding || + context.binding->children.size() < elem_serdes_ptrs.size()) { + return Status::InvalidArgument("BSON STRUCT requires pinned field bindings"); + } + const bool timestamp = value.value.value_type == BSON_TYPE_TIMESTAMP; + const bool regex = value.value.value_type == BSON_TYPE_REGEX; + if (!timestamp && !regex && value.value.value_type != BSON_TYPE_DOCUMENT) { + return Status::InvalidArgument("BSON STRUCT requires a document or timestamp source"); + } + auto& target = assert_cast(column); + const size_t before = target.size(); + bool committed = false; + Defer rollback([&] { + if (!committed) { + // A failure can occur after an earlier child appended, including an allocation exception. + for (size_t j = 0; j < elem_serdes_ptrs.size(); ++j) { + auto& child = target.get_column(j); + if (child.size() > before) { + child.pop_back(child.size() - before); + } + } + } + }); + for (size_t i = 0; i < elem_serdes_ptrs.size(); ++i) { + // Nested projection can retain only city from STRUCT; child ordinals then differ. + const BsonBinding* binding = context.binding->child(elem_names[i], i); + Status status; + BsonValueView field; + if (!binding) { + status = Status::InvalidArgument("BSON STRUCT field differs from the pinned binding"); + } else if (timestamp || regex) { + bson_value_t part {}; + part.value_type = timestamp ? BSON_TYPE_INT64 : BSON_TYPE_UTF8; + bool valid = true; + if (regex) { + const char* text = nullptr; + if (binding->name == "pattern") { + text = value.value.value.v_regex.regex; + } else if (binding->name == "options") { + text = value.value.value.v_regex.options; + } + valid = text != nullptr; + if (text) { + part.value.v_utf8.str = const_cast(text); + part.value.v_utf8.len = static_cast(strlen(text)); + } + } else if (binding->name == "seconds") { + part.value.v_int64 = value.value.value.v_timestamp.timestamp; + } else if (binding->name == "increment") { + part.value.v_int64 = value.value.value.v_timestamp.increment; + } else { + valid = false; + } + field = BsonValueView::of(part); + status = valid ? elem_serdes_ptrs[i]->read_one_cell_from_bson( + target.get_column(i), field, context.child(binding)) + : Status::InvalidArgument( + "Invalid BSON timestamp or regex field binding"); + } else { + field = value.at_path(binding->path); + status = elem_serdes_ptrs[i]->read_one_cell_from_bson(target.get_column(i), field, + context.child(binding)); + } + if (!status.ok()) { + return status; + } + } + committed = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_struct_serde.h b/be/src/core/data_type_serde/data_type_struct_serde.h index 1cfa3ee6c74db7..90e3cc3e7a0019 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.h +++ b/be/src/core/data_type_serde/data_type_struct_serde.h @@ -39,6 +39,9 @@ class DataTypeStructSerDe : public DataTypeSerDe { elem_serdes_ptrs(_elem_serdes_ptrs), elem_names(names) {} + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; + std::string get_name() const override; Status from_string(StringRef& str, IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp index e8ee0e5a473a73..1f395dc0030b64 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp @@ -25,6 +25,7 @@ #include "core/column/column_varbinary.h" #include "core/data_type/data_type_varbinary.h" #include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/bson_value_view.h" #include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "exprs/function/string_hex_util.h" @@ -441,4 +442,32 @@ void DataTypeVarbinarySerDe::to_string(const IColumn& column, size_t row_num, Bu } } +Status DataTypeVarbinarySerDe::decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const { + RETURN_IF_ERROR(context.validate(value)); + if (value.missing) { + return Status::InvalidArgument("Missing BSON binary value"); + } + const auto& v = value.value; + // VARBINARY length is schema metadata, as in the other readers: VARBINARY(4) must still + // preserve all 12 ObjectId bytes rather than reject or truncate the value. + auto& target = assert_cast(column); + if (context.binding && context.binding->special == "bson" && + v.value_type == BSON_TYPE_DOCUMENT) { + target.insert_data(reinterpret_cast(v.value.v_doc.data), + v.value.v_doc.data_len); + } else if (v.value_type == BSON_TYPE_OID) { + target.insert_data(reinterpret_cast(v.value.v_oid.bytes), 12); + } else if (v.value_type == BSON_TYPE_BINARY && context.binding && + context.binding->subtype >= 0 && + context.binding->subtype == static_cast(v.value.v_binary.subtype)) { + target.insert_data(reinterpret_cast(v.value.v_binary.data), + v.value.v_binary.data_len); + } else { + return Status::InvalidArgument( + "BSON binary type or subtype differs from the pinned schema"); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index 7413cdbe32fc3b..e6737ee992951d 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -34,7 +34,10 @@ namespace doris { class DataTypeVarbinarySerDe : public DataTypeSerDe { public: - DataTypeVarbinarySerDe(int nesting_level = 1) : DataTypeSerDe(nesting_level) {}; + DataTypeVarbinarySerDe(int nesting_level = 1) : DataTypeSerDe(nesting_level) {} + + Status decode_bson_value(IColumn& column, const BsonValueView& value, + const BsonDecodeContext& context) const override; std::string get_name() const override { return "Varbinary"; } diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 8efe0bc599cbc3..86d261b3397cb0 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -68,6 +68,7 @@ #include "format_v2/table/iceberg_reader.h" #include "format_v2/table/lance_reader.h" #include "format_v2/table/load_reader.h" +#include "format_v2/table/mongodb_reader.h" #include "format_v2/table/paimon_reader.h" #include "format_v2/table/remote_doris_reader.h" #include "format_v2/table_reader.h" @@ -678,6 +679,8 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(mc_desc); } else if (table_format == "trino_connector") { *reader = std::make_unique(); + } else if (table_format == "mongodb") { + *reader = std::make_unique(); } else if (table_format == "es") { *reader = std::make_unique(); } else if (table_format == "remote_doris") { @@ -968,6 +971,9 @@ Status FileScannerV2::_to_file_format(TFileFormatType::type format_type, case TFileFormatType::FORMAT_ARROW: *file_format = format::FileFormat::ARROW; return Status::OK(); + case TFileFormatType::FORMAT_MONGODB: + *file_format = format::FileFormat::MONGODB; + return Status::OK(); case TFileFormatType::FORMAT_ES_HTTP: *file_format = format::FileFormat::ES_HTTP; return Status::OK(); diff --git a/be/src/format/CMakeLists.txt b/be/src/format/CMakeLists.txt index a729cd486dd92f..02d6885f877f03 100644 --- a/be/src/format/CMakeLists.txt +++ b/be/src/format/CMakeLists.txt @@ -27,6 +27,7 @@ file(GLOB_RECURSE FORMAT_V2_SRC_FILES CONFIGURE_DEPENDS list(APPEND SRC_FILES ${FORMAT_V2_SRC_FILES}) add_library(Format STATIC ${SRC_FILES}) +target_link_libraries(Format PUBLIC mongoc2) # Read the pinned SDK timezone rules without changing SDK decoding or public interfaces. target_include_directories(Format PRIVATE diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 1ef84f79dee5ad..aa3a4756dc9c34 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -61,6 +61,7 @@ enum class FileFormat { ARROW, WAL, ES_HTTP, + MONGODB, LANCE, }; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 40f8e07d853dac..e27d984770981f 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -158,6 +158,8 @@ const char* file_format_name(FileFormat format) { return "JNI"; case FileFormat::ARROW: return "ARROW"; + case FileFormat::MONGODB: + return "MONGODB"; case FileFormat::ES_HTTP: return "ES_HTTP"; case FileFormat::WAL: diff --git a/be/src/format_v2/table/mongodb_reader.cpp b/be/src/format_v2/table/mongodb_reader.cpp new file mode 100644 index 00000000000000..cad5c054f038e7 --- /dev/null +++ b/be/src/format_v2/table/mongodb_reader.cpp @@ -0,0 +1,520 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/mongodb_reader.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/data_type_serde/bson_value_view.h" +#include "core/data_type_serde/data_type_serde.h" +#include "format_v2/column_mapper.h" +#include "format_v2/materialized_reader_util.h" +#include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::mongodb { +namespace { +using Clock = std::chrono::steady_clock; +template +struct MongoDeleter { + template + void operator()(T* value) const { + if (value) { + Destroy(value); + } + } +}; +template +using MongoPtr = std::unique_ptr>; +using UriPtr = MongoPtr; +using CollectionPtr = MongoPtr; +using CursorPtr = MongoPtr; +using ReadPrefsPtr = MongoPtr; +using ReadConcernPtr = MongoPtr; + +struct ClientPool { + MongoPtr pool; +}; + +Status acquire_pool(const TMongoScanDesc& d, std::shared_ptr* result) { + // Driver initialization is process-wide; cleanup while another scanner is alive is unsafe. + static std::once_flag initialized; + std::call_once(initialized, [] { mongoc_init(); }); + static std::mutex mutex; + static std::unordered_map> pools; + // Include exact credentials as well as the FE version: even a malformed descriptor cannot reuse + // another identity's authenticated client. The key is private memory and is never logged. + std::string key = std::to_string(d.catalog_id) + ":" + d.configuration_version; + for (const auto* value : {&d.uri, &d.username, &d.password, &d.auth_database}) { + key += ":" + std::to_string(value->size()) + ":" + *value; + } + key += ":" + std::to_string(d.timeout_ms) + ":" + std::to_string(d.connect_timeout_ms) + ":" + + std::to_string(d.server_selection_timeout_ms) + ":" + + std::to_string(d.socket_timeout_ms) + ":" + std::to_string(d.pool_max_clients); + { + std::lock_guard lock(mutex); + for (auto it = pools.begin(); it != pools.end();) { + it = it->second.expired() ? pools.erase(it) : std::next(it); + } + if (auto found = pools.find(key); found != pools.end()) { + if (auto pool = found->second.lock()) { + *result = std::move(pool); + return Status::OK(); + } + } + } + bson_error_t error {}; + UriPtr uri(mongoc_uri_new_with_error(d.uri.c_str(), &error)); + if (!uri) { + return Status::InvalidArgument("Invalid MongoDB connection configuration"); + } + if (!d.username.empty() && (!mongoc_uri_set_username(uri.get(), d.username.c_str()) || + !mongoc_uri_set_password(uri.get(), d.password.c_str()) || + !mongoc_uri_set_auth_source(uri.get(), d.auth_database.c_str()))) { + return Status::InvalidArgument("Invalid MongoDB authentication configuration"); + } + mongoc_uri_set_option_as_int32(uri.get(), MONGOC_URI_MAXPOOLSIZE, d.pool_max_clients); + mongoc_uri_set_option_as_int32( + uri.get(), MONGOC_URI_CONNECTTIMEOUTMS, + static_cast(std::min(d.connect_timeout_ms, d.timeout_ms))); + mongoc_uri_set_option_as_int32( + uri.get(), MONGOC_URI_SERVERSELECTIONTIMEOUTMS, + static_cast(std::min(d.server_selection_timeout_ms, d.timeout_ms))); + mongoc_uri_set_option_as_int32(uri.get(), MONGOC_URI_SOCKETTIMEOUTMS, d.socket_timeout_ms); + mongoc_uri_set_option_as_bool(uri.get(), MONGOC_URI_RETRYREADS, false); + auto pool = std::make_shared(); + pool->pool.reset(mongoc_client_pool_new_with_error(uri.get(), &error)); + if (!pool->pool) { + return Status::InvalidArgument("MongoDB client pool initialization failed"); + } + mongoc_client_pool_set_error_api(pool->pool.get(), 2); + { + std::lock_guard lock(mutex); + auto& existing = pools[key]; + if (auto shared = existing.lock()) { + *result = std::move(shared); + } else { + existing = pool; + *result = std::move(pool); + } + } + return Status::OK(); +} + +Status mongo_error(const bson_error_t& error) { + // Driver messages can contain URIs, credentials, commands, or source values. + return Status::IOError("MongoDB command failed (domain={}, code={})", error.domain, error.code); +} + +class NativeMongoStream final : public MongoCursorStream { +public: + ~NativeMongoStream() override { close(); } + Status open(const TMongoScanDesc& d, std::function cancelled) override { + _cancelled = std::move(cancelled); + _deadline = Clock::now() + std::chrono::milliseconds(d.timeout_ms); + RETURN_IF_ERROR(check_deadline()); + RETURN_IF_ERROR(acquire_pool(d, &_pool)); + while (!(_client = mongoc_client_pool_try_pop(_pool->pool.get()))) { + RETURN_IF_ERROR(check_deadline()); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + mongoc_client_set_sockettimeoutms(_client, static_cast(std::min( + d.socket_timeout_ms, remaining_ms()))); + RETURN_IF_ERROR(verify_collection_identity(d)); + _collection.reset( + mongoc_client_get_collection(_client, d.database.c_str(), d.collection.c_str())); + if (!_collection) { + return Status::InvalidArgument("Invalid MongoDB namespace"); + } + ReadConcernPtr concern(mongoc_read_concern_new()); + if (!mongoc_read_concern_set_level(concern.get(), d.read_concern.c_str())) { + return Status::InvalidArgument("Invalid MongoDB read concern"); + } + mongoc_collection_set_read_concern(_collection.get(), concern.get()); + RETURN_IF_ERROR(configure_mongodb_read_preference(_collection.get(), d.read_preference)); + bson_t filter = BSON_INITIALIZER; + bson_t options = BSON_INITIALIZER; + BSON_APPEND_INT32(&options, "batchSize", d.batch_size); + BSON_APPEND_INT64(&options, "maxTimeMS", remaining_ms()); + // Always query the public logical namespace: a view may change rows, names, and _id, + // and time-series buckets are an internal storage representation. + _cursor.reset( + mongoc_collection_find_with_opts(_collection.get(), &filter, &options, nullptr)); + bson_destroy(&options); + bson_destroy(&filter); + if (!_cursor) { + return Status::IOError("MongoDB cursor initialization failed"); + } + return check_deadline(); + } + Status next(const bson_t** document, bool* eof) override { + RETURN_IF_ERROR(check_deadline()); + if (mongoc_cursor_next(_cursor.get(), document)) { + *eof = false; + return check_deadline(); + } + bson_error_t error {}; + if (mongoc_cursor_error(_cursor.get(), &error)) { + return mongo_error(error); + } + *eof = true; + return Status::OK(); + } + void close() override { + // Destroy the cursor (killCursors) before releasing its collection and exclusive pool lease. + _cursor.reset(); + _collection.reset(); + if (_client) { + mongoc_client_pool_push(_pool->pool.get(), _client); + _client = nullptr; + } + _pool.reset(); + } + +private: + Status verify_collection_identity(const TMongoScanDesc& d) { + if (d.collection_uuid.empty()) { + return Status::OK(); + } + MongoPtr database( + mongoc_client_get_database(_client, d.database.c_str())); + ReadPrefsPtr primary(mongoc_read_prefs_new(MONGOC_READ_PRIMARY)); + mongoc_database_set_read_prefs(database.get(), primary.get()); + ReadConcernPtr metadata_concern(mongoc_read_concern_new()); + mongoc_database_set_read_concern(database.get(), metadata_concern.get()); + bson_t options = BSON_INITIALIZER; + BSON_APPEND_INT64(&options, "maxTimeMS", remaining_ms()); + bson_t filter; + BSON_APPEND_DOCUMENT_BEGIN(&options, "filter", &filter); + BSON_APPEND_UTF8(&filter, "name", d.collection.c_str()); + bson_append_document_end(&options, &filter); + BSON_APPEND_INT32(&options, "batchSize", 1); + CursorPtr cursor(mongoc_database_find_collections_with_opts(database.get(), &options)); + bson_destroy(&options); + const bson_t* document = nullptr; + std::string actual; + if (cursor && mongoc_cursor_next(cursor.get(), &document)) { + auto identity = BsonValueView::document(*document).at_path({"info", "uuid"}); + if (!identity.missing && identity.value.value_type == BSON_TYPE_BINARY && + identity.value.value.v_binary.subtype == BSON_SUBTYPE_UUID) { + const auto& bytes = identity.value.value.v_binary; + static constexpr char digits[] = "0123456789abcdef"; + for (uint32_t i = 0; i < bytes.data_len; ++i) { + actual += digits[bytes.data[i] >> 4]; + actual += digits[bytes.data[i] & 15]; + } + } + } + bson_error_t error {}; + if (!cursor || mongoc_cursor_error(cursor.get(), &error)) { + return mongo_error(error); + } + // Namespace reuse after DROP/CREATE must fail instead of scanning an unrelated collection. + if (actual != d.collection_uuid) { + return Status::InvalidArgument( + "MongoDB collection identity changed; refresh the catalog and retry"); + } + return check_deadline(); + } + int64_t remaining_ms() const { + return std::max( + 1, std::chrono::duration_cast(_deadline - Clock::now()) + .count()); + } + Status check_deadline() const { + if (_cancelled && _cancelled()) { + return Status::Cancelled("MongoDB scan cancelled"); + } + if (Clock::now() >= _deadline) { + return Status::TimedOut("MongoDB scan deadline exceeded"); + } + return Status::OK(); + } + std::function _cancelled; + Clock::time_point _deadline; + std::shared_ptr _pool; + mongoc_client_t* _client = nullptr; + CollectionPtr _collection; + CursorPtr _cursor; +}; +} // namespace + +std::unique_ptr make_native_mongo_stream() { + return std::make_unique(); +} + +Status configure_mongodb_read_preference(_mongoc_collection_t* collection, + const std::string& preference) { + mongoc_read_mode_t mode = MONGOC_READ_PRIMARY; + if (preference == "primaryPreferred") { + mode = MONGOC_READ_PRIMARY_PREFERRED; + } else if (preference == "secondary") { + mode = MONGOC_READ_SECONDARY; + } else if (preference == "secondaryPreferred") { + mode = MONGOC_READ_SECONDARY_PREFERRED; + } else if (preference == "nearest") { + mode = MONGOC_READ_NEAREST; + } else if (preference != "primary") { + return Status::InvalidArgument("Invalid MongoDB read preference"); + } + // The collection inherited the complete URI preference. A new mode-only preference loses + // tag sets and max staleness; copy it before applying FE's effective mode instead. + ReadPrefsPtr prefs(mongoc_read_prefs_copy(mongoc_collection_get_read_prefs(collection))); + mongoc_read_prefs_set_mode(prefs.get(), mode); + // For example, a malformed descriptor must not apply primary together with secondary tags. + if (!mongoc_read_prefs_is_valid(prefs.get())) { + return Status::InvalidArgument("Incompatible MongoDB read preference criteria"); + } + mongoc_collection_set_read_prefs(collection, prefs.get()); + return Status::OK(); +} + +Status validate_mongodb_range(const TFileRangeDesc& range) { + if (!range.__isset.table_format_params || + range.table_format_params.table_format_type != "mongodb" || + !range.table_format_params.__isset.mongodb_params) { + return Status::InvalidArgument("MongoDB native reader requires mongodb_params"); + } + const auto& d = range.table_format_params.mongodb_params; + if (!d.__isset.version || d.version != 1 || d.database.empty() || d.collection.empty() || + d.collection.starts_with("system.") || d.database.find('\0') != std::string::npos || + d.collection.find('\0') != std::string::npos || d.uri.empty() || d.schema_version.empty() || + d.configuration_version.empty() || d.bindings_json.empty() || + (d.object_kind != "collection" && d.object_kind != "view" && + d.object_kind != "timeseries") || + d.batch_size < 1 || d.batch_size > 65536 || d.timeout_ms < 1 || d.connect_timeout_ms < 1 || + d.connect_timeout_ms > 300000 || d.server_selection_timeout_ms < 1 || + d.server_selection_timeout_ms > 300000 || d.socket_timeout_ms < 1 || + d.socket_timeout_ms > 300000 || d.pool_max_clients < 1 || d.pool_max_clients > 1024 || + (d.read_concern != "local" && d.read_concern != "majority" && + d.read_concern != "available")) { + return Status::InvalidArgument("Invalid or unsupported MongoDB native scan descriptor"); + } + return Status::OK(); +} + +MongoFileReader::MongoFileReader(std::shared_ptr& properties, + std::unique_ptr& description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + TFileRangeDesc range, std::vector slots, + MongoStreamFactory factory) + : FileReader(properties, description, std::move(io_ctx), profile), + _range(std::move(range)), + _slots(std::move(slots)), + _factory(std::move(factory)) {} +MongoFileReader::~MongoFileReader() { + (void)close(); +} +void MongoFileReader::_init_profile() { + if (!_profile) { + return; + } + _io_time = file_scan_profile::ensure_hierarchy(_profile).io; + const char* parent = "MongoFileReader"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, parent, file_scan_profile::FILE_READER, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "MongoReadTime", parent, 1); + _decode_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "MongoDecodeTime", parent, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "MongoFilterTime", parent, 1); + _rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MongoRowsRead", TUnit::UNIT, parent, 1); + _bytes_read = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MongoBsonBytes", TUnit::BYTES, parent, 1); +} +Status MongoFileReader::init(RuntimeState* state) { + _state = state; + RETURN_IF_ERROR(validate_mongodb_range(_range)); + RETURN_IF_ERROR(parse_bson_bindings(_range.table_format_params.mongodb_params.bindings_json, + &_bindings)); + _batch_size = state->batch_size(); + _init_profile(); + _eof = false; + return Status::OK(); +} +Status MongoFileReader::get_schema(std::vector* schema) const { + schema->clear(); + for (size_t i = 0; i < _slots.size(); ++i) { + const auto* slot = _slots[i]; + schema->push_back({.identifier = Field::create_field(cast_set(i)), + .local_id = cast_set(i), + .name = slot->col_name(), + .type = slot->type(), + .children = materialized_schema_children(slot->type())}); + } + return Status::OK(); +} +std::unique_ptr MongoFileReader::create_column_mapper( + TableColumnMapperOptions options) const { + return std::make_unique(options); +} +void MongoFileReader::set_batch_size(size_t size) { + DORIS_CHECK(size > 0); + _batch_size = size; +} +Status MongoFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + if (_stream) { + return Status::InvalidArgument("MongoDB reader already has an open cursor"); + } + _read_bindings.clear(); + _positions.clear(); + _serdes.clear(); + _eof = false; + RETURN_IF_ERROR(FileReader::open(std::move(request))); + // Request iteration order need not match slot order. Keep binding/SerDe entries aligned with + // their file-local block positions, including reordered or pruned projections. + for (const auto& [id, position] : _request->local_positions) { + if (id.value() < 0 || static_cast(id.value()) >= _slots.size()) { + return Status::InvalidArgument("Invalid MongoDB projected slot"); + } + const auto* slot = _slots[id.value()]; + auto binding = std::ranges::find_if( + _bindings, [&](const auto& b) { return b.name == slot->col_name(); }); + if (binding == _bindings.end()) { + return Status::InvalidArgument( + "MongoDB projected column differs from the pinned schema"); + } + _read_bindings.push_back(&*binding); + _positions.push_back(position.value()); + _serdes.push_back(slot->type()->get_serde()); + } + _stream = _factory ? _factory() : make_native_mongo_stream(); + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_read_time); + Status status = _stream->open(_range.table_format_params.mongodb_params, [this] { + return _state->is_cancelled() || (_io_ctx && _io_ctx->should_stop); + }); + if (!status.ok()) { + (void)close(); + } + return status; +} +Status MongoFileReader::get_block(Block* block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); + *rows = 0; + *eof = _eof; + if (_eof) { + return Status::OK(); + } + Status status = _read_block(block, rows, eof); + if (!status.ok() || *eof) { + (void)close(); + } + return status; +} +Status MongoFileReader::_read_block(Block* block, size_t* rows, bool* eof) { + { + auto guard = block->mutate_columns_scoped(); + auto& columns = guard.mutable_columns(); + std::vector sizes(_positions.size()); + size_t bson_bytes = 0; + while (*rows < _batch_size) { + RETURN_IF_CANCELLED(_state); + if (_io_ctx && _io_ctx->should_stop) { + *eof = true; + break; + } + const bson_t* document = nullptr; + bool source_eof = false; + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_read_time); + RETURN_IF_ERROR(_stream->next(&document, &source_eof)); + } + if (source_eof) { + *eof = true; + break; + } + if (!document || document->len > BsonDecodeContext::MAX_DOCUMENT_BYTES || + !bson_validate(document, BSON_VALIDATE_NONE, nullptr)) { + return Status::Corruption("MongoDB cursor returned invalid BSON"); + } + SCOPED_TIMER(_decode_time); + BsonValueView root = BsonValueView::document(*document); + // Resource/structural failures apply to the entire document, outside the per-field + // mismatch-as-NULL policy, so projection cannot bypass the decoding limits. + RETURN_IF_ERROR(validate_bson_tree(root)); + // A later field can fail after earlier columns appended. Snapshot every column before + // the first append so the failing source document contributes no partial Doris row. + for (size_t i = 0; i < _positions.size(); ++i) { + sizes[i] = columns[_positions[i]]->size(); + } + for (size_t i = 0; i < _positions.size(); ++i) { + const auto* binding = _read_bindings[i]; + BsonValueView value = binding->special.empty() ? root.at_path(binding->path) : root; + BsonDecodeContext context { + .binding = binding, + .mismatch_as_null = + _range.table_format_params.mongodb_params.type_mismatch_as_null}; + Status status = _serdes[i]->read_one_cell_from_bson(*columns[_positions[i]], value, + context); + if (!status.ok()) { + for (size_t j = 0; j < _positions.size(); ++j) { + auto& column = *columns[_positions[j]]; + column.pop_back(column.size() - sizes[j]); + } + return status; + } + } + // All SerDes have copied borrowed payloads; the next stream advance may release them. + ++*rows; + if (_bytes_read) { + COUNTER_UPDATE(_bytes_read, document->len); + } + bson_bytes += document->len; + // A row-count limit alone can create a multi-GB block for large documents. Allow one + // oversized document, then yield; the source itself bounds each BSON document to 16 MiB. + if (bson_bytes >= 8 * 1024 * 1024) { + break; + } + } + } + _record_scan_rows(cast_set(*rows)); + if (_rows_read) { + COUNTER_UPDATE(_rows_read, *rows); + } + SCOPED_TIMER(_filter_time); + return apply_materialized_reader_filters(_request.get(), _io_ctx.get(), block, rows); +} +Status MongoFileReader::close() { + auto stream = std::move(_stream); + if (stream) { + stream->close(); + } + _request.reset(); + _eof = true; + return Status::OK(); +} +MongoReader::MongoReader(MongoStreamFactory factory) : _factory(std::move(factory)) {} +Status MongoReader::prepare_split(const SplitReadOptions& options) { + RETURN_IF_ERROR(validate_mongodb_range(options.current_range)); + return TableReader::prepare_split(options); +} +Status MongoReader::create_file_reader(std::unique_ptr* reader) { + *reader = std::make_unique(_system_properties, _current_task->data_file, + _io_ctx, _scanner_profile, _current_file_range_desc, + *_file_slot_descs, _factory); + return Status::OK(); +} + +} // namespace doris::format::mongodb diff --git a/be/src/format_v2/table/mongodb_reader.h b/be/src/format_v2/table/mongodb_reader.h new file mode 100644 index 00000000000000..db7a744378b7e8 --- /dev/null +++ b/be/src/format_v2/table/mongodb_reader.h @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "format_v2/file_reader.h" +#include "format_v2/table_reader.h" + +struct _bson_t; +struct _mongoc_collection_t; +namespace doris { +struct BsonBinding; +class DataTypeSerDe; +class SlotDescriptor; +} // namespace doris + +namespace doris::format::mongodb { + +// A cursor owns the borrowed BSON document until next() or close(). No cursor crosses a split boundary. +class MongoCursorStream { +public: + virtual ~MongoCursorStream() = default; + // Open the logical namespace with a private client lease. The callback stays valid until close; + // callers must close even after a partially successful open returns an error. + virtual Status open(const TMongoScanDesc& descriptor, std::function cancelled) = 0; + // On success, either return one borrowed document with eof=false, or eof=true with no document. + // A command/cancellation error is a failing Status, never normal EOF. + virtual Status next(const _bson_t** document, bool* eof) = 0; + // Idempotently destroy the cursor before returning its client lease to the pool. + virtual void close() = 0; +}; +// Production libmongoc transport; a factory can inject a deterministic stream into reader tests. +std::unique_ptr make_native_mongo_stream(); +using MongoStreamFactory = std::function()>; +// Validate the typed single-cursor contract and resource bounds before creating driver objects. +Status validate_mongodb_range(const TFileRangeDesc& range); +// Apply the effective catalog mode while preserving URI tag sets and max staleness on the collection. +Status configure_mongodb_read_preference(_mongoc_collection_t* collection, + const std::string& preference); + +// FileScannerV2 adapter for one MongoDB logical namespace. init parses the pinned schema without +// network I/O; open binds a file-local request and creates its stream; get_block copies BSON values +// into Doris columns through their SerDes. EOF/error/early termination all end in close. +// Scanner execution is serial for this reader; RuntimeState and slot descriptors outlive it. +class MongoFileReader final : public FileReader { +public: + MongoFileReader(std::shared_ptr& properties, + std::unique_ptr& description, + std::shared_ptr io_ctx, RuntimeProfile* profile, + TFileRangeDesc range, std::vector slots, + MongoStreamFactory factory = {}); + ~MongoFileReader() override; + // Freeze bindings from the FE descriptor and initialize session batch limits/profile counters. + Status init(RuntimeState* state) override; + // Expose projected slot types as the materialized file schema, with slot ordinals as local IDs. + Status get_schema(std::vector* schema) const override; + // Keep table-to-local identity translation in the mapper; the reader consumes its local request. + std::unique_ptr create_column_mapper( + TableColumnMapperOptions options) const override; + // Override the Doris output row limit; MongoDB's driver batch_size remains in the descriptor. + void set_batch_size(size_t size) override; + // Resolve local column bindings and open a fresh cursor. A second open needs an intervening close. + Status open(std::shared_ptr request) override; + // Produce locally filtered rows, releasing the cursor on EOF/error. An empty block can still + // have eof=false when local predicates remove the entire source batch. + Status get_block(Block* block, size_t* rows, bool* eof) override; + // Release the cursor/request and mark EOF. Repeated close and destructor cleanup are safe. + Status close() override; + +private: + // Attribute native transport time to the common FileReader I/O hierarchy as well as Mongo timers. + void _init_profile() override; + // Append whole rows until the row/BSON-byte budget is reached, then evaluate localized filters. + // SerDes roll back partial cells; this method also rolls back earlier columns in a failed row. + Status _read_block(Block* block, size_t* rows, bool* eof); + const TFileRangeDesc _range; + const std::vector _slots; + MongoStreamFactory _factory; + RuntimeState* _state = nullptr; + size_t _batch_size = 1024; + std::unique_ptr _stream; + // Owned by the reader and immutable after init, keeping _read_bindings pointers stable. + std::vector _bindings; + // Parallel vectors indexed by projected field: binding, file-local block position and target SerDe. + std::vector _read_bindings; + std::vector _positions; + std::vector> _serdes; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeProfile::Counter* _read_time = nullptr; + RuntimeProfile::Counter* _decode_time = nullptr; + RuntimeProfile::Counter* _rows_read = nullptr; + RuntimeProfile::Counter* _bytes_read = nullptr; +}; + +// TableReader integration retains generic split orchestration and final table-block materialization. +class MongoReader final : public TableReader { +public: + explicit MongoReader(MongoStreamFactory factory = {}); + Status prepare_split(const SplitReadOptions& options) override; + +protected: + Status create_file_reader(std::unique_ptr* reader) override; + +private: + MongoStreamFactory _factory; +}; + +} // namespace doris::format::mongodb diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 1f1e5604f63e70..0c25797633e58e 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -111,6 +111,8 @@ std::string file_format_to_string(FileFormat format) { return "JNI"; case FileFormat::ARROW: return "ARROW"; + case FileFormat::MONGODB: + return "MONGODB"; case FileFormat::ES_HTTP: return "ES_HTTP"; case FileFormat::WAL: diff --git a/be/test/core/data_type_serde/data_type_serde_bson_test.cpp b/be/test/core/data_type_serde/data_type_serde_bson_test.cpp new file mode 100644 index 00000000000000..0810757d20eee4 --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_bson_test.cpp @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#ifdef ADDRESS_SANITIZER +#include +#endif + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_jsonb.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/bson_value_view.h" +#include "core/data_type_serde/data_type_number_serde.h" +#include "core/value/vdatetime_value.h" +#include "util/jsonb_utils.h" + +namespace doris { +namespace { +#ifdef ADDRESS_SANITIZER +thread_local bool measure_bson_allocations = false; +thread_local size_t largest_bson_allocation = 0; +void bson_allocation_hook(const volatile void*, size_t bytes) { + if (measure_bson_allocations) { + largest_bson_allocation = std::max(largest_bson_allocation, bytes); + } +} +void bson_free_hook(const volatile void*) {} +#endif +BsonBinding binding(std::string name = "v", std::vector tags = {}) { + BsonBinding result; + result.name = name; + result.path = {name}; + result.types = std::move(tags); + return result; +} +class BsonSerdeTest : public testing::Test { +protected: + Status read(const DataTypePtr& type, IColumn& column, const std::string& json, + const BsonBinding& field, bool null_on_error = false) { + bson_error_t error {}; + bson_t* document = bson_new_from_json(reinterpret_cast(json.data()), + json.size(), &error); + if (!document) { + return Status::InvalidArgument("Invalid test BSON"); + } + auto value = BsonValueView::document(*document).at_path(field.path); + auto status = type->get_serde()->read_one_cell_from_bson( + column, value, {.binding = &field, .mismatch_as_null = null_on_error}); + bson_destroy(document); + return status; + } + Status read(const DataTypePtr& type, IColumn& column, const std::string& json, + bool null_on_error = false) { + return read(type, column, json, binding(), null_on_error); + } +}; +} // namespace + +TEST_F(BsonSerdeTest, IntegerWidthsAreExactAndOverflowDoesNotAppend) { + auto type = std::make_shared(); + auto column = type->create_column(); + ASSERT_TRUE(read(type, *column, R"({"v":{"$numberLong":"9223372036854775807"}})").ok()); + ASSERT_TRUE(read(type, *column, R"({"v":{"$numberLong":"-9223372036854775808"}})").ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], std::numeric_limits::max()); + EXPECT_EQ(data[1], std::numeric_limits::min()); + EXPECT_FALSE(read(type, *column, R"({"v":1.5})").ok()); + EXPECT_FALSE(read(type, *column, R"({"v":"123"})").ok()); + EXPECT_EQ(column->size(), 2); + auto narrow = std::make_shared(); + auto small = narrow->create_column(); + EXPECT_TRUE(read(narrow, *small, R"({"v":127})").ok()); + EXPECT_FALSE(read(narrow, *small, R"({"v":128})").ok()); + EXPECT_EQ(small->size(), 1); +} +TEST_F(BsonSerdeTest, BooleanAndDoubleKeepTheirPhysicalTags) { + auto boolean = std::make_shared(); + auto c = boolean->create_column(); + EXPECT_TRUE(read(boolean, *c, R"({"v":true})").ok()); + EXPECT_FALSE(read(boolean, *c, R"({"v":1})").ok()); + auto type = std::make_shared(); + auto d = type->create_column(); + EXPECT_TRUE(read(type, *d, R"({"v":{"$numberDouble":"NaN"}})").ok()); + EXPECT_TRUE(read(type, *d, R"({"v":{"$numberDouble":"Infinity"}})").ok()); + EXPECT_FALSE(read(type, *d, R"({"v":2})").ok()); + const auto& values = assert_cast(*d).get_data(); + EXPECT_TRUE(std::isnan(values[0])); + EXPECT_TRUE(std::isinf(values[1])); +} +TEST_F(BsonSerdeTest, MissingAndNullRemainDistinctFromUndefined) { + auto type = make_nullable(std::make_shared()); + auto column = type->create_column(); + EXPECT_TRUE(read(type, *column, "{}").ok()); + EXPECT_TRUE(read(type, *column, R"({"v":null})").ok()); + EXPECT_FALSE(read(type, *column, R"({"v":{"$undefined":true}})").ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_TRUE(assert_cast(*column).is_null_at(0)); +} +TEST_F(BsonSerdeTest, LiteralDotAndNestedPathsDoNotAlias) { + auto type = std::make_shared(); + auto column = type->create_column(); + BsonBinding field = binding("a.b"); + ASSERT_TRUE(read(type, *column, R"({"a.b":1,"a":{"b":2}})", field).ok()); + field.path = {"a", "b"}; + ASSERT_TRUE(read(type, *column, R"({"a.b":1,"a":{"b":2}})", field).ok()); + EXPECT_EQ(assert_cast(*column).get_data()[0], 1); + EXPECT_EQ(assert_cast(*column).get_data()[1], 2); +} +TEST_F(BsonSerdeTest, StringCopiesBorrowedStorageAndPreservesEmbeddedNull) { + auto type = std::make_shared(); + auto column = type->create_column(); + ASSERT_TRUE(read(type, *column, R"({"v":"a\u0000b"})").ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("a\0b", 3)); + EXPECT_FALSE(read(type, *column, R"({"v":{"a":1}})").ok()); +} +TEST_F(BsonSerdeTest, ObjectIdIsTwelveBytesAndNotInterchangeableWithBinData) { + auto type = std::make_shared(12); + auto column = type->create_column(); + auto field = binding("v", {7}); + ASSERT_TRUE(read(type, *column, R"({"v":{"$oid":"000102030405060708090a0b"}})", field).ok()); + std::string expected; + for (int i = 0; i < 12; ++i) { + expected += static_cast(i); + } + EXPECT_EQ(column->get_data_at(0).to_string(), expected); + ASSERT_TRUE(read(type, *column, R"({"v":{"$oid":"00ff807f0100020304050607"}})", field).ok()); + const std::string high_bytes("\x00\xff\x80\x7f\x01\x00\x02\x03\x04\x05\x06\x07", 12); + EXPECT_EQ(column->get_data_at(1).to_string(), high_bytes); + EXPECT_FALSE(read(type, *column, + R"({"v":{"$binary":{"base64":"AAECAwQFBgcICQoL","subType":"00"}}})", field) + .ok()); +} +TEST_F(BsonSerdeTest, BinarySubtypeIsCheckedAndEmptyBinaryIsNotNull) { + auto type = make_nullable(std::make_shared()); + auto column = type->create_column(); + auto field = binding("v", {5}); + field.subtype = 128; + ASSERT_TRUE( + read(type, *column, R"({"v":{"$binary":{"base64":"","subType":"80"}}})", field).ok()); + EXPECT_FALSE(assert_cast(*column).is_null_at(0)); + EXPECT_FALSE( + read(type, *column, R"({"v":{"$binary":{"base64":"","subType":"00"}}})", field).ok()); + EXPECT_EQ(column->size(), 1); +} +TEST_F(BsonSerdeTest, DecimalScaleNeverSilentlyRounds) { + auto type = std::make_shared>(10, 2); + auto column = type->create_column(); + for (const auto& value : {"123.4500", "-1.2E+2", "0E-6176", "1E-2"}) { + EXPECT_TRUE( + read(type, *column, "{\"v\":{\"$numberDecimal\":\"" + std::string(value) + "\"}}") + .ok()); + } + for (const auto& value : {"1.234", "100000000.00", "NaN", "Infinity", "1E+6144"}) { + EXPECT_FALSE( + read(type, *column, "{\"v\":{\"$numberDecimal\":\"" + std::string(value) + "\"}}") + .ok()); + } + EXPECT_EQ(column->size(), 4); +} +TEST_F(BsonSerdeTest, NegativeEpochMillisecondsUseFloorDivision) { + auto type = std::make_shared(3); + auto column = type->create_column(); + ASSERT_TRUE(read(type, *column, R"({"v":{"$date":{"$numberLong":"-1"}}})").ok()); + auto packed = assert_cast(*column).get_data()[0]; + DateV2Value value; + memcpy(&value, &packed, sizeof(value)); + EXPECT_EQ(value.year(), 1969); + EXPECT_EQ(value.month(), 12); + EXPECT_EQ(value.day(), 31); + EXPECT_EQ(value.second(), 59); + EXPECT_EQ(value.microsecond(), 999000); + EXPECT_FALSE( + read(type, *column, R"({"v":{"$date":{"$numberLong":"9223372036854775807"}}})").ok()); + auto seconds = std::make_shared(0); + auto other = seconds->create_column(); + EXPECT_FALSE(read(seconds, *other, R"({"v":{"$date":{"$numberLong":"-1"}}})").ok()); +} +TEST_F(BsonSerdeTest, ArrayFailureRollsBackNestedDataBeforeNullPolicy) { + auto type = make_nullable( + std::make_shared(make_nullable(std::make_shared()))); + auto column = type->create_column(); + auto field = binding("v", {4}); + field.children = {binding("element", {16})}; + ASSERT_TRUE(read(type, *column, R"({"v":[1,2]})", field).ok()); + ASSERT_FALSE(read(type, *column, R"({"v":[3,"bad"]})", field).ok()); + EXPECT_EQ(column->size(), 1); + const auto& array = + assert_cast(assert_cast(*column).get_nested_column()); + EXPECT_EQ(array.get_data().size(), 2); + // A failed array is one SQL NULL; partial elements must not escape into the next row. + ASSERT_TRUE(read(type, *column, R"({"v":[3,"bad",4]})", field, true).ok()); + EXPECT_EQ(array.get_data().size(), 2); + EXPECT_TRUE(assert_cast(*column).is_null_at(1)); + ASSERT_TRUE(read(type, *column, R"({"v":{}})", field, true).ok()); + EXPECT_TRUE(assert_cast(*column).is_null_at(2)); +} +TEST_F(BsonSerdeTest, StructUsesFieldNamesAndRollsBackEveryChild) { + DataTypes children {make_nullable(std::make_shared()), + make_nullable(std::make_shared())}; + auto type = std::make_shared(children, Strings {"a", "b"}); + auto column = type->create_column(); + auto field = binding("v", {3}); + field.children = {binding("a", {16}), binding("b", {2})}; + ASSERT_TRUE(read(type, *column, R"({"v":{"b":"ok","a":1,"extra":true}})", field).ok()); + ASSERT_FALSE(read(type, *column, R"({"v":{"a":2,"b":3}})", field).ok()); + auto& structure = assert_cast(*column); + EXPECT_EQ(structure.get_column(0).size(), 1); + EXPECT_EQ(structure.get_column(1).size(), 1); + ASSERT_TRUE(read(type, *column, R"({"v":{"a":3}})", field).ok()); + EXPECT_TRUE(assert_cast(structure.get_column(1)).is_null_at(1)); +} +TEST_F(BsonSerdeTest, TimestampRetainsUnsignedSecondsAndIncrement) { + auto integer = make_nullable(std::make_shared()); + auto type = std::make_shared(DataTypes {integer, integer}, + Strings {"seconds", "increment"}); + auto column = type->create_column(); + auto field = binding("v", {17}); + field.children = {binding("seconds", {18}), binding("increment", {18})}; + ASSERT_TRUE( + read(type, *column, R"({"v":{"$timestamp":{"t":4294967295,"i":4294967295}}})", field) + .ok()); + const auto& child = + assert_cast(assert_cast(*column).get_column(0)); + EXPECT_EQ(assert_cast(child.get_nested_column()).get_data()[0], + 4294967295LL); +} +TEST_F(BsonSerdeTest, CanonicalExtendedJsonRetainsAllSpecialBsonTags) { + auto type = std::make_shared(); + auto column = type->create_column(); + std::vector values { + R"({"$numberLong":"9007199254740993"})", + R"({"$numberDecimal":"123.4500"})", + R"({"$oid":"000102030405060708090a0b"})", + R"({"$date":{"$numberLong":"-1"}})", + R"({"$binary":{"base64":"AA==","subType":"80"}})", + R"({"$regularExpression":{"pattern":"a.*","options":"i"}})", + R"({"$timestamp":{"t":42,"i":7}})", + R"({"$minKey":1})", + R"({"$maxKey":1})", + R"({"$undefined":true})", + R"({"$dbPointer":{"$ref":"sample.records","$id":{"$oid":"000102030405060708090a0b"}}})", + R"({"$code":"return 1"})", + R"({"$symbol":"s"})", + R"({"$code":"return n","$scope":{"n":1}})"}; + for (const auto& value : values) { + ASSERT_TRUE(read(type, *column, "{\"v\":" + value + "}").ok()) << value; + } + auto first = column->get_data_at(0); + EXPECT_NE(JsonbToJson::jsonb_to_json_string(first.data, first.size).find("9007199254740993"), + std::string::npos); + EXPECT_EQ(column->size(), values.size()); +} +TEST_F(BsonSerdeTest, DecodeDepthAndDescriptorLimitsAreEnforced) { + auto type = std::make_shared(); + auto column = type->create_column(); + bson_value_t v {}; + v.value_type = BSON_TYPE_INT32; + v.value.v_int32 = 1; + auto field = binding(); + EXPECT_FALSE( + type->get_serde() + ->read_one_cell_from_bson(*column, BsonValueView::of(v), {&field, false, 65}) + .ok()); + std::vector fields; + EXPECT_FALSE(parse_bson_bindings("{}", &fields).ok()); + EXPECT_FALSE(parse_bson_bindings(std::string(1024 * 1024 + 1, 'x'), &fields).ok()); + EXPECT_FALSE( + parse_bson_bindings(R"({"columns":[{"name":"v","path":[],"bson_types":[],"children":[]}, + {"name":"v","path":[],"bson_types":[],"children":[]}]})", + &fields) + .ok()); +} + +TEST_F(BsonSerdeTest, WideDocumentValidationUsesBoundedScratch) { + bson_t document = BSON_INITIALIZER; + bson_array_builder_t* array = nullptr; + ASSERT_TRUE(BSON_APPEND_ARRAY_BUILDER_BEGIN(&document, "wide", &array)); + bson_t empty = BSON_INITIALIZER; + for (uint32_t i = 0; i < 900000; ++i) { + ASSERT_TRUE(bson_array_builder_append_document(array, &empty)); + } + ASSERT_TRUE(bson_append_array_builder_end(&document, array)); + ASSERT_LT(document.len, BsonDecodeContext::MAX_DOCUMENT_BYTES); +#ifdef ADDRESS_SANITIZER + // Measure only validation scratch: the source BSON is already allocated. Ordinary vectors + // bypass MemTracker, so source/output row budgets cannot detect this allocation regression. + static const int installed = + __sanitizer_install_malloc_and_free_hooks(bson_allocation_hook, bson_free_hook); + ASSERT_EQ(installed, 1); + largest_bson_allocation = 0; + measure_bson_allocations = true; +#endif + Status status = validate_bson_tree(BsonValueView::document(document)); +#ifdef ADDRESS_SANITIZER + measure_bson_allocations = false; + EXPECT_LT(largest_bson_allocation, 64 * 1024); +#endif + EXPECT_TRUE(status.ok()) << status; + bson_destroy(&empty); + bson_destroy(&document); +} + +TEST_F(BsonSerdeTest, TreeDepthLimitIncludesDocumentsArraysAndCodeScopes) { + for (auto kind : {BSON_TYPE_DOCUMENT, BSON_TYPE_ARRAY, BSON_TYPE_CODEWSCOPE}) { + bson_t nested = BSON_INITIALIZER; + for (int depth = 0; depth <= BsonDecodeContext::MAX_DEPTH; ++depth) { + if (depth == BsonDecodeContext::MAX_DEPTH) { + EXPECT_TRUE(validate_bson_tree(BsonValueView::document(nested)).ok()); + } + bson_t parent = BSON_INITIALIZER; + bool appended; + if (kind == BSON_TYPE_DOCUMENT) { + appended = BSON_APPEND_DOCUMENT(&parent, "0", &nested); + } else if (kind == BSON_TYPE_ARRAY) { + appended = BSON_APPEND_ARRAY(&parent, "0", &nested); + } else { + appended = BSON_APPEND_CODE_WITH_SCOPE(&parent, "0", "", &nested); + } + ASSERT_TRUE(appended); + bson_destroy(&nested); + bson_copy_to(&parent, &nested); + bson_destroy(&parent); + } + EXPECT_FALSE(validate_bson_tree(BsonValueView::document(nested)).ok()); + bson_destroy(&nested); + } +} + +TEST_F(BsonSerdeTest, TreeElementLimitIsGlobalAndInclusive) { + bson_t document = BSON_INITIALIZER; + for (uint32_t i = 0; i < BsonDecodeContext::MAX_ELEMENTS; ++i) { + char key[16]; + const char* key_ptr = nullptr; + bson_uint32_to_string(i, &key_ptr, key, sizeof(key)); + ASSERT_TRUE(bson_append_int32(&document, key_ptr, -1, 1)); + } + ASSERT_LT(document.len, BsonDecodeContext::MAX_DOCUMENT_BYTES); + EXPECT_TRUE(validate_bson_tree(BsonValueView::document(document)).ok()); + ASSERT_TRUE(BSON_APPEND_INT32(&document, "extra", 1)); + EXPECT_FALSE(validate_bson_tree(BsonValueView::document(document)).ok()); + bson_destroy(&document); +} + +TEST_F(BsonSerdeTest, RawDocumentBytesRoundTripWithoutJsonOrBorrowedStorage) { + auto type = std::make_shared(1); + auto column = type->create_column(); + bson_t document = BSON_INITIALIZER; + BSON_APPEND_INT64(&document, "exact", 9007199254740993LL); + BsonBinding field; + field.special = "bson"; + std::string bytes(reinterpret_cast(bson_get_data(&document)), document.len); + ASSERT_TRUE( + type->get_serde() + ->read_one_cell_from_bson(*column, BsonValueView::document(document), {&field}) + .ok()); + bson_destroy(&document); + EXPECT_EQ(column->get_data_at(0).to_string(), bytes); +} +TEST_F(BsonSerdeTest, VarbinaryLengthIsMetadataForBsonPayloads) { + auto type = make_nullable(std::make_shared(4)); + auto column = type->create_column(); + auto field = binding("v", {7}); + ASSERT_TRUE(read(type, *column, R"({"v":{"$oid":"000102030405060708090a0b"}})", field).ok()); + ASSERT_TRUE( + read(type, *column, R"({"v":{"$oid":"000102030405060708090a0b"}})", field, true).ok()); + const auto& nullable = assert_cast(*column); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_EQ(nullable.get_nested_column().get_data_at(0).to_string(), + std::string("\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 12)); + field = binding("v", {5}); + field.subtype = 128; + ASSERT_TRUE( + read(type, *column, R"({"v":{"$binary":{"base64":"AAECAwQF","subType":"80"}}})", field) + .ok()); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(nullable.get_nested_column().get_data_at(2).to_string(), + std::string("\x00\x01\x02\x03\x04\x05", 6)); +} +TEST_F(BsonSerdeTest, NullableBinarySubtypeMismatchStillUsesConfiguredPolicy) { + auto type = make_nullable(std::make_shared(4)); + auto column = type->create_column(); + auto field = binding("v", {5}); + field.subtype = 0; + EXPECT_FALSE(read(type, *column, R"({"v":{"$binary":{"base64":"AA==","subType":"80"}}})", field) + .ok()); + ASSERT_TRUE(read(type, *column, R"({"v":{"$binary":{"base64":"AA==","subType":"80"}}})", field, + true) + .ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_TRUE(assert_cast(*column).is_null_at(0)); +} +TEST_F(BsonSerdeTest, RegexStructPreservesPatternAndOptionsWithoutExecutingThem) { + auto text = make_nullable(std::make_shared()); + auto type = std::make_shared(DataTypes {text, text}, + Strings {"pattern", "options"}); + auto column = type->create_column(); + auto field = binding("v", {11}); + field.children = {binding("pattern", {2}), binding("options", {2})}; + ASSERT_TRUE(read(type, *column, + R"({"v":{"$regularExpression":{"pattern":"a.*b","options":"im"}}})", field) + .ok()); + const auto& target = assert_cast(*column); + EXPECT_EQ(assert_cast(target.get_column(0)) + .get_nested_column() + .get_data_at(0) + .to_string(), + "a.*b"); + EXPECT_EQ(assert_cast(target.get_column(1)) + .get_nested_column() + .get_data_at(0) + .to_string(), + "im"); +} +TEST_F(BsonSerdeTest, NestedArrayFailureDoesNotAlterTheFollowingRow) { + auto integer = make_nullable(std::make_shared()); + auto inner = make_nullable(std::make_shared(integer)); + auto type = make_nullable(std::make_shared(inner)); + auto column = type->create_column(); + auto field = binding("v", {4}); + auto child = binding("element", {4}); + child.children = {binding("element", {16})}; + field.children = {child}; + ASSERT_TRUE(read(type, *column, R"({"v":[[1,2],[3,"bad"]]})", field, true).ok()); + ASSERT_TRUE(read(type, *column, R"({"v":[[],[4,null]]})", field, true).ok()); + const auto& top = assert_cast(*column); + EXPECT_TRUE(top.is_null_at(0)); + EXPECT_FALSE(top.is_null_at(1)); + const auto& outer = assert_cast(top.get_nested_column()); + const auto& nullable_inner = assert_cast(outer.get_data()); + const auto& arrays = assert_cast(nullable_inner.get_nested_column()); + EXPECT_EQ(arrays.get_data().size(), 2); +} +TEST_F(BsonSerdeTest, DeepExtendedJsonIsRejectedBeforeRecursiveEncoding) { + std::string json = "1"; + for (int i = 0; i < 70; ++i) { + json = "[" + json + "]"; + } + auto type = std::make_shared(); + auto column = type->create_column(); + EXPECT_FALSE(read(type, *column, "{\"v\":" + json + "}").ok()); + EXPECT_EQ(column->size(), 0); +} +TEST_F(BsonSerdeTest, ExplicitEpochMillisecondsAvoidsDatetimeRangeLoss) { + auto type = std::make_shared(); + auto column = type->create_column(); + auto field = binding("v", {9}); + ASSERT_TRUE( + read(type, *column, R"({"v":{"$date":{"$numberLong":"9223372036854775807"}}})", field) + .ok()); + EXPECT_EQ(assert_cast(*column).get_data()[0], + std::numeric_limits::max()); +} + +TEST_F(BsonSerdeTest, StructProjectionUsesNamesAfterPruningAndReordering) { + auto type = std::make_shared( + DataTypes {make_nullable(std::make_shared())}, Strings {"b"}); + auto column = type->create_column(); + auto field = binding("v", {3}); + field.children = {binding("a", {16}), binding("b", {2})}; + field.child_positions = {{"a", 0}, {"b", 1}}; + ASSERT_TRUE(read(type, *column, R"({"v":{"b":"projected","a":123}})", field).ok()); + const auto& target = assert_cast(*column); + EXPECT_EQ(assert_cast(target.get_column(0)) + .get_nested_column() + .get_data_at(0) + .to_string(), + "projected"); +} + +TEST_F(BsonSerdeTest, ScalarAncestorIsATypeMismatchRatherThanMissing) { + auto type = make_nullable(std::make_shared()); + auto column = type->create_column(); + auto field = binding("v"); + field.path = {"parent", "child"}; + EXPECT_FALSE(read(type, *column, R"({"parent":1})", field).ok()); + EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(read(type, *column, R"({"parent":1})", field, true).ok()); + EXPECT_TRUE(read(type, *column, R"({"parent":null})", field).ok()); + EXPECT_TRUE(read(type, *column, "{}", field).ok()); + EXPECT_EQ(column->size(), 3); +} + +TEST_F(BsonSerdeTest, HeterogeneousArrayCanPreserveBsonTagsAsJson) { + auto type = std::make_shared(make_nullable(std::make_shared())); + auto column = type->create_column(); + auto field = binding("v", {4}); + field.children = {binding("element")}; + ASSERT_TRUE( + read(type, *column, + R"({"v":[{"$oid":"000102030405060708090a0b"},null,{"n":{"$numberLong":"9007199254740993"}}]})", + field) + .ok()); + const auto& array = assert_cast(*column); + const auto& nested = assert_cast(array.get_data()); + EXPECT_EQ(nested.size(), 3); + EXPECT_TRUE(nested.is_null_at(1)); + auto raw = nested.get_nested_column().get_data_at(2); + EXPECT_NE(JsonbToJson::jsonb_to_json_string(raw.data, raw.size).find("9007199254740993"), + std::string::npos); +} + +TEST_F(BsonSerdeTest, AllocationFailureIsNotConvertedToNullAndRollsBackAppends) { + class ThrowingSerDe final : public DataTypeNumberSerDe { + public: + Status decode_bson_value(IColumn& column, const BsonValueView&, + const BsonDecodeContext&) const override { + column.insert_default(); + throw Exception(ErrorCode::MEM_ALLOC_FAILED, "fixture allocation failure"); + } + }; + auto type = make_nullable(std::make_shared()); + auto column = type->create_column(); + DataTypeNullableSerDe serde(std::make_shared()); + bson_value_t v {}; + v.value_type = BSON_TYPE_INT32; + v.value.v_int32 = 1; + auto field = binding("v", {16}); + auto status = serde.read_one_cell_from_bson(*column, BsonValueView::of(v), + {.binding = &field, .mismatch_as_null = true}); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.code(), ErrorCode::INVALID_ARGUMENT); + EXPECT_EQ(column->size(), 0); + EXPECT_EQ(assert_cast(*column).get_nested_column().size(), 0); +} + +TEST_F(BsonSerdeTest, DecimalStorageWidthsPreserveExactScaledValues) { + auto t32 = std::make_shared>(9, 2); + auto c32 = t32->create_column(); + ASSERT_TRUE(read(t32, *c32, R"({"v":{"$numberDecimal":"123.4500"}})").ok()); + EXPECT_EQ(assert_cast&>(*c32).get_data()[0].value, 12345); + auto t128 = std::make_shared>(38, 10); + auto c128 = t128->create_column(); + ASSERT_TRUE(read(t128, *c128, R"({"v":{"$numberDecimal":"123.4500"}})").ok()); + EXPECT_EQ(assert_cast&>(*c128).get_data()[0].value, + static_cast(1234500000000LL)); +} + +} // namespace doris diff --git a/be/test/format_v2/table/mongodb_reader_test.cpp b/be/test/format_v2/table/mongodb_reader_test.cpp new file mode 100644 index 00000000000000..31e7b4e8d998cd --- /dev/null +++ b/be/test/format_v2/table/mongodb_reader_test.cpp @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/mongodb_reader.h" + +#include +#include +#include + +#include "common/object_pool.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type_serde/bson_value_view.h" +#include "format_v2/column_mapper.h" +#include "runtime/runtime_state.h" +#include "testutil/desc_tbl_builder.h" + +namespace doris::format::mongodb { +namespace { +struct StreamState { + int opens = 0; + int closes = 0; + int reads = 0; + bool fail_open = false; + bool fail_read = false; + std::string kind; + std::vector documents {R"({"id":1,"value":10})", R"({"id":2,"value":20})", + R"({"id":3,"value":30})"}; +}; +class FixtureStream final : public MongoCursorStream { +public: + explicit FixtureStream(std::shared_ptr state) : _state(std::move(state)) {} + ~FixtureStream() override { + if (_document) { + bson_destroy(_document); + } + } + Status open(const TMongoScanDesc& d, std::function cancelled) override { + ++_state->opens; + _state->kind = d.object_kind; + _cancelled = std::move(cancelled); + return _state->fail_open ? Status::IOError("fixture open failed") : Status::OK(); + } + Status next(const bson_t** document, bool* eof) override { + ++_state->reads; + if (_cancelled()) { + return Status::Cancelled("fixture cancelled"); + } + if (_state->fail_read) { + return Status::IOError("fixture getMore failed"); + } + if (_document) { + bson_destroy(_document); + _document = nullptr; + } + *eof = _position == _state->documents.size(); + if (!*eof) { + const auto& json = _state->documents[_position++]; + _document = bson_new_from_json(reinterpret_cast(json.data()), + json.size(), nullptr); + if (!_document) { + return Status::Corruption("invalid fixture"); + } + *document = _document; + } + return Status::OK(); + } + void close() override { ++_state->closes; } + +private: + std::shared_ptr _state; + std::function _cancelled; + size_t _position = 0; + bson_t* _document = nullptr; +}; +TFileRangeDesc range() { + TMongoScanDesc d; + d.__set_version(1); + d.__set_catalog_id(1); + d.__set_configuration_version("fixture-config"); + d.__set_database("sample"); + d.__set_collection("records"); + d.__set_object_kind("collection"); + d.__set_schema_version("fixture-schema"); + d.__set_uri("mongodb://localhost:27017"); + d.__set_read_preference("primary"); + d.__set_read_concern("majority"); + d.__set_batch_size(2); + d.__set_timeout_ms(30000); + d.__set_connect_timeout_ms(1000); + d.__set_server_selection_timeout_ms(1000); + d.__set_socket_timeout_ms(1000); + d.__set_pool_max_clients(2); + d.__set_bindings_json(R"({"version":1,"columns":[ + {"name":"id","path":["id"],"bson_types":[16],"children":[]}, + {"name":"value","path":["value"],"bson_types":[16],"children":[]}]})"); + TTableFormatFileDesc table; + table.__set_table_format_type("mongodb"); + table.__set_mongodb_params(d); + TFileRangeDesc result; + result.__set_format_type(TFileFormatType::FORMAT_MONGODB); + result.__set_path("/dummyPath"); + result.__set_table_format_params(table); + return result; +} +class MongoReaderTest : public testing::Test { +protected: + void SetUp() override { + DescriptorTblBuilder builder(&_pool); + builder.declare_tuple() << std::make_tuple(std::make_shared(), + std::string("id")) + << std::make_tuple(std::make_shared(), + std::string("value")); + _slots = builder.build()->get_tuple_descriptor(0)->slots(); + } + std::unique_ptr reader() { + auto props = std::make_shared(); + auto file = std::make_unique(); + auto result = std::make_unique( + props, file, _io, &_profile, _range, _slots, + [state = _responses] { return std::make_unique(state); }); + EXPECT_TRUE(result->init(&_state).ok()); + result->set_batch_size(2); + return result; + } + std::shared_ptr request(std::initializer_list ids) { + auto result = std::make_shared(); + FileScanRequestBuilder builder(result.get()); + for (int id : ids) { + EXPECT_TRUE(builder.add_non_predicate_column(LocalColumnId(id)).ok()); + } + return result; + } + Block block(const MongoFileReader& reader, const FileScanRequest& request) { + std::vector schema; + EXPECT_TRUE(reader.get_schema(&schema).ok()); + ColumnsWithTypeAndName columns(request.block_column_count()); + for (auto [id, position] : request.local_positions) { + const auto& column = schema[id.value()]; + columns[position.value()] = {column.type->create_column(), column.type, column.name}; + } + return {columns}; + } + ObjectPool _pool; + RuntimeState _state; + RuntimeProfile _profile {"mongo_reader_test"}; + std::shared_ptr _io = std::make_shared(); + TFileRangeDesc _range = range(); + std::vector _slots; + std::shared_ptr _responses = std::make_shared(); +}; +} // namespace + +TEST_F(MongoReaderTest, BatchesAndProjectionOrderUseNativeBson) { + auto scan = reader(); + auto projection = request({1, 0}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + EXPECT_FALSE(eof); + auto position = projection->local_positions.at(LocalColumnId(1)).value(); + const auto& value = + assert_cast(*output.get_by_position(position).column); + EXPECT_EQ(assert_cast(value.get_nested_column()).get_data()[1], 20); + output = block(*scan, *projection); + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 1); + EXPECT_TRUE(eof); + EXPECT_EQ(_responses->closes, 1); + ASSERT_TRUE(scan->close().ok()); + EXPECT_EQ(_responses->closes, 1); +} +TEST_F(MongoReaderTest, EveryLogicalObjectKindKeepsItsNamespace) { + for (const auto& kind : {"collection", "view", "timeseries"}) { + _range.table_format_params.mongodb_params.__set_object_kind(kind); + auto scan = reader(); + ASSERT_TRUE(scan->open(request({0})).ok()); + EXPECT_EQ(_responses->kind, kind); + ASSERT_TRUE(scan->close().ok()); + } + EXPECT_EQ(_responses->closes, 3); +} +TEST_F(MongoReaderTest, OpenAndGetMoreErrorsReleaseStreamExactlyOnce) { + _responses->fail_open = true; + { + auto scan = reader(); + EXPECT_FALSE(scan->open(request({0})).ok()); + } + EXPECT_EQ(_responses->closes, 1); + _responses->fail_open = false; + _responses->fail_read = true; + { + auto scan = reader(); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(scan->get_block(&output, &rows, &eof).ok()); + } + EXPECT_EQ(_responses->closes, 2); +} +TEST_F(MongoReaderTest, StoppingBeforeReadReleasesCursor) { + auto scan = reader(); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + _io->should_stop = true; + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + EXPECT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(_responses->reads, 0); + EXPECT_EQ(_responses->closes, 1); +} +TEST_F(MongoReaderTest, RowConversionErrorRollsBackPreviouslyWrittenColumns) { + _responses->documents = {R"({"id":1,"value":"wrong"})"}; + auto scan = reader(); + auto projection = request({0, 1}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + EXPECT_FALSE(scan->get_block(&output, &rows, &eof).ok()); + for (size_t i = 0; i < output.columns(); ++i) { + EXPECT_EQ(output.get_by_position(i).column->size(), 0); + } + EXPECT_EQ(_responses->closes, 1); +} +TEST_F(MongoReaderTest, NullMismatchPolicyPreservesRowAndOtherFields) { + _range.table_format_params.mongodb_params.__set_type_mismatch_as_null(true); + _responses->documents = {R"({"id":1,"value":"wrong"})", R"({"id":2,"value":20})"}; + auto scan = reader(); + auto projection = request({0, 1}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + auto pos = projection->local_positions.at(LocalColumnId(1)).value(); + const auto& column = assert_cast(*output.get_by_position(pos).column); + EXPECT_TRUE(column.is_null_at(0)); + EXPECT_FALSE(column.is_null_at(1)); +} +TEST_F(MongoReaderTest, EmptyAndDestructorPathsReleaseCursor) { + _responses->documents.clear(); + { + auto scan = reader(); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + EXPECT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_TRUE(eof); + EXPECT_EQ(rows, 0); + } + EXPECT_EQ(_responses->closes, 1); + { + auto scan = reader(); + ASSERT_TRUE(scan->open(request({0})).ok()); + } + EXPECT_EQ(_responses->closes, 2); +} +TEST_F(MongoReaderTest, InvalidDescriptorsAndInternalBucketsAreRejected) { + EXPECT_FALSE(validate_mongodb_range(TFileRangeDesc()).ok()); + for (const auto& name : {"system.buckets.measurements", "system.views"}) { + auto invalid = range(); + invalid.table_format_params.mongodb_params.__set_collection(name); + EXPECT_FALSE(validate_mongodb_range(invalid).ok()); + } + auto invalid = range(); + invalid.table_format_params.mongodb_params.__set_batch_size(0); + EXPECT_FALSE(validate_mongodb_range(invalid).ok()); + invalid = range(); + invalid.table_format_params.mongodb_params.__set_read_concern("snapshot"); + EXPECT_FALSE(validate_mongodb_range(invalid).ok()); +} + +TEST_F(MongoReaderTest, BsonByteBudgetBoundsBlocksWithLargeUnprojectedFields) { + const std::string payload(4 * 1024 * 1024, 'x'); + _responses->documents.clear(); + for (int i = 0; i < 3; ++i) { + _responses->documents.push_back(R"({"id":1,"payload":")" + payload + "\"}"); + } + auto scan = reader(); + scan->set_batch_size(100); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + EXPECT_FALSE(eof); + output = block(*scan, *projection); + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 1); + EXPECT_TRUE(eof); +} +TEST_F(MongoReaderTest, ReopeningAfterCloseDoesNotDuplicateProjectedColumns) { + auto scan = reader(); + auto projection = request({0}); + ASSERT_TRUE(scan->open(projection).ok()); + EXPECT_FALSE(scan->open(projection).ok()); + ASSERT_TRUE(scan->close().ok()); + ASSERT_TRUE(scan->open(projection).ok()); + auto output = block(*scan, *projection); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(scan->get_block(&output, &rows, &eof).ok()); + EXPECT_EQ(rows, 2); + EXPECT_EQ(output.rows(), 2); + ASSERT_TRUE(scan->close().ok()); + EXPECT_EQ(_responses->closes, 2); +} + +TEST(MongoNativeStreamTest, ReadPreferencePreservesUriTagsAndMaxStaleness) { + mongoc_init(); + std::unique_ptr uri( + mongoc_uri_new("mongodb://localhost:27017/?readPreference=secondary&" + "readPreferenceTags=region:analytics&readPreferenceTags=&" + "maxStalenessSeconds=90"), + mongoc_uri_destroy); + ASSERT_NE(uri, nullptr); + std::unique_ptr client( + mongoc_client_new_from_uri(uri.get()), mongoc_client_destroy); + std::unique_ptr collection( + mongoc_client_get_collection(client.get(), "sample", "records"), + mongoc_collection_destroy); + ASSERT_TRUE(configure_mongodb_read_preference(collection.get(), "secondary").ok()); + const auto* prefs = mongoc_collection_get_read_prefs(collection.get()); + EXPECT_EQ(mongoc_read_prefs_get_mode(prefs), MONGOC_READ_SECONDARY); + EXPECT_EQ(mongoc_read_prefs_get_max_staleness_seconds(prefs), 90); + EXPECT_TRUE(bson_equal(mongoc_read_prefs_get_tags(prefs), + mongoc_read_prefs_get_tags(mongoc_uri_get_read_prefs_t(uri.get())))); + EXPECT_FALSE(configure_mongodb_read_preference(collection.get(), "invalid").ok()); + EXPECT_FALSE(configure_mongodb_read_preference(collection.get(), "primary").ok()); + EXPECT_EQ(mongoc_read_prefs_get_mode(mongoc_collection_get_read_prefs(collection.get())), + MONGOC_READ_SECONDARY); +} + +TEST(MongoNativeStreamTest, CatalogModeAppliesWhenUriHasNoRoutingCriteria) { + mongoc_init(); + std::unique_ptr client( + mongoc_client_new("mongodb://localhost:27017"), mongoc_client_destroy); + std::unique_ptr collection( + mongoc_client_get_collection(client.get(), "sample", "records"), + mongoc_collection_destroy); + for (const auto& [name, mode] : std::vector> { + {"primary", MONGOC_READ_PRIMARY}, + {"primaryPreferred", MONGOC_READ_PRIMARY_PREFERRED}, + {"secondary", MONGOC_READ_SECONDARY}, + {"secondaryPreferred", MONGOC_READ_SECONDARY_PREFERRED}, + {"nearest", MONGOC_READ_NEAREST}}) { + ASSERT_TRUE(configure_mongodb_read_preference(collection.get(), name).ok()); + EXPECT_EQ(mongoc_read_prefs_get_mode(mongoc_collection_get_read_prefs(collection.get())), + mode); + } +} + +TEST(MongoNativeStreamTest, CancellationBeforeOpenDoesNotConnect) { + auto stream = make_native_mongo_stream(); + auto descriptor = range().table_format_params.mongodb_params; + EXPECT_EQ(stream->open(descriptor, [] { return true; }).code(), ErrorCode::CANCELLED); + stream->close(); + stream->close(); +} +TEST(MongoNativeStreamTest, InvalidUriIsRedactedAndPartialOpenCanBeClosed) { + auto stream = make_native_mongo_stream(); + auto descriptor = range().table_format_params.mongodb_params; + descriptor.__set_uri("fixture-secret-invalid-uri"); + auto status = stream->open(descriptor, [] { return false; }); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("fixture-secret"), std::string::npos); + stream->close(); +} +TEST(MongoNativeStreamTest, FailedConnectionReturnsErrorAndReleasesLeases) { + for (int attempt = 0; attempt < 3; ++attempt) { + auto stream = make_native_mongo_stream(); + auto descriptor = range().table_format_params.mongodb_params; + descriptor.__set_uri("mongodb://fixture-user:fixture-secret@127.0.0.1:1"); + descriptor.__set_timeout_ms(100); + descriptor.__set_server_selection_timeout_ms(25); + descriptor.__set_connect_timeout_ms(25); + descriptor.__set_socket_timeout_ms(25); + auto status = stream->open(descriptor, [] { return false; }); + if (status.ok()) { + const bson_t* document = nullptr; + bool eof = false; + status = stream->next(&document, &eof); + } + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.to_string().find("fixture-secret"), std::string::npos); + EXPECT_EQ(status.to_string().find("mongodb://"), std::string::npos); + stream->close(); + } +} + +} // namespace doris::format::mongodb diff --git a/build.sh b/build.sh index 279a6f0ad2ca69..ef53d46c870417 100755 --- a/build.sh +++ b/build.sh @@ -832,14 +832,12 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Keep this list identical to the deploy loop's (search CONN_PLUGIN_DIR). A module missing here # but present there is not a no-op: the deploy step unzips whatever archive is left in the # module's target/ from some earlier build, so the plugin silently ships stale. - for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss; do + for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb; do if [[ -d "${DORIS_HOME}/fe/fe-connector/fe-connector-${_conn_mod}" ]]; then modules+=("fe-connector/fe-connector-${_conn_mod}") fi done unset _conn_mod - # Build the dependency bundle before the native connector provider is implemented. - modules+=("fe-connector/fe-connector-mongodb") # Authorization plugin modules (loaded at runtime from plugins/authorization/). Keep this list # identical to the deploy loop's (search AUTHZ_PLUGIN_DIR), for the same reason as the connectors: # the deploy step unzips whatever archive is left in the module's target/, so a module built here @@ -1350,11 +1348,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy connector provider plugins as independent plugin directories. # Each sub-directory is one connector backend loaded at runtime by ConnectorPluginManager. CONN_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/connector" - # Keep this dependency-only archive out of auto-discovery, which requires a provider class. - mkdir -p "${CONN_PLUGIN_DIR}" - cp -p "${DORIS_HOME}/fe/fe-connector/fe-connector-mongodb/target/doris-fe-connector-mongodb.zip" \ - "${CONN_PLUGIN_DIR}/mongodb.zip" - for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss; do + for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg adbc fluss mongodb; do conn_plugin_target="${CONN_PLUGIN_DIR}/${conn_module}" conn_module_dir="${DORIS_HOME}/fe/fe-connector/fe-connector-${conn_module}" if [ ! -d "${conn_module_dir}" ]; then diff --git a/docker/thirdparties/docker-compose/mongodb/README.md b/docker/thirdparties/docker-compose/mongodb/README.md index 11609a834ec67b..47bfb662cf1b3d 100644 --- a/docker/thirdparties/docker-compose/mongodb/README.md +++ b/docker/thirdparties/docker-compose/mongodb/README.md @@ -1,6 +1,6 @@ # MongoDB regression fixture -This optional standalone MongoDB 8.0.20 service supplies deterministic BSON data for the embedded Trino 435 MongoDB connector. It does not require a Trino server. Start it with: +This optional standalone MongoDB 8.0.20 service supplies deterministic BSON data for the native MongoDB Catalog and the embedded Trino 435 MongoDB connector. Start it with: ```bash CONTAINER_UID=doris-mongodb-test- bash docker/thirdparties/run-thirdparties-docker.sh -c mongodb @@ -53,3 +53,29 @@ WHERE json_extract_int(payload, '$.n') = 7; ``` Automatic schema inference still maps BSON documents to Trino `row` / Doris `STRUCT`. Use an explicit `json` declaration when a field should be read as a whole JSON value. BSON documents use the connector's Extended JSON representation, so embedded ObjectIds appear as `{"$oid":"..."}` objects. For BSON string values declared as `json`, their contents must be valid JSON text; malformed text fails the query. The JSON suite covers both real BSON documents and valid JSON text, plus a malformed-text negative case. + +## Native MongoDB Catalog + +`02-native-fixtures.js` initializes the separate `doris_mongodb_native` database. +Its fixture contains validator-based collections, a computed view without `_id`, +a time-series collection with duplicate timestamps, BSON special values, malformed +historical values, and a reader granted `find` only on the view. These tests use +the native FE plugin and C++ reader and need no Trino plugin installation. + +Use the same regression configuration above, then run: + +```bash +bash run-regression-test.sh --run -d external_table_p0/mongodb -s test_mongodb_native +``` + +The suite covers auto/explicit/document schemas, exact ObjectId and int64 values, +Decimal128 scales, UTC dates, binary subtypes, nested arrays/structs, canonical +Extended JSON, missing/null/mismatch handling, SQL filters and LIKE/REGEXP, +aggregations, joins, windows, multi-batch reads, repeated early termination, +views, time-series measurements and view-only authorization. Legal tuple/pattern/ +dependency validators verify complete document fallback. A 900,000-container BSON +fixture exercises scalar projection through whole-document validation; the separate +`doris_mongodb_pagination` database has 140 empty collections to cross the metadata +cursor's 128-name batch boundary. Binary displays and +analysis use `FROM_BINARY`; direct VARBINARY operations keep Doris's existing +analysis restrictions. diff --git a/docker/thirdparties/docker-compose/mongodb/init/02-native-fixtures.js b/docker/thirdparties/docker-compose/mongodb/init/02-native-fixtures.js new file mode 100644 index 00000000000000..d8015af76aae8f --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/init/02-native-fixtures.js @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const nativeFixture = db.getSiblingDB("doris_mongodb_native"); +nativeFixture.dropDatabase(); +const nativeOid = n => ObjectId(n.toString(16).padStart(24, "0")); +nativeFixture.createCollection("typed_records", {validator: {$jsonSchema: { + bsonType: "object", required: ["seq"], properties: { + _id: {bsonType: "objectId"}, seq: {bsonType: "long"}, name: {bsonType: "string"}, + grp: {bsonType: "string"}, score: {bsonType: ["int", "long", "null"]}, + active: {bsonType: "bool"}, ratio: {bsonType: "double"}, + amount: {bsonType: "decimal"}, created: {bsonType: "date"}, + tags: {bsonType: "array", items: {bsonType: ["string", "null"]}}, + profile: {bsonType: "object", properties: { + City: {bsonType: "string"}, rank: {bsonType: "long"}, manager: {bsonType: "objectId"} + }}, + clock: {bsonType: "timestamp"}, blob: {bsonType: "binData"} + } +}}}); +// mongosh encodes integral JavaScript numbers as Int32; Double keeps the validator's physical tag. +nativeFixture.typed_records.insertMany([ + {_id: nativeOid(1), seq: NumberLong("1"), name: "alpha", grp: "east", score: NumberInt(10), + active: true, ratio: 1.25, amount: NumberDecimal("123.4500"), created: new Date(-1), + tags: ["red", "blue"], profile: {City: "A", rank: NumberLong("1"), manager: nativeOid(101)}, + clock: Timestamp({t: 4294967295, i: 4294967295}), blob: BinData(0, "AP+A")}, + {_id: nativeOid(2), seq: NumberLong("2"), name: "beta", grp: "west", score: NumberLong("20"), + active: false, ratio: -0.5, amount: NumberDecimal("-2.50"), created: ISODate("2024-02-01T12:34:56.789Z"), + tags: [], profile: {City: "B", rank: NumberLong("2")}, blob: BinData(128, "AP+A")}, + {_id: nativeOid(3), seq: NumberLong("3"), name: "中文-é", grp: "east", score: null, + active: true, ratio: Double(0.0), amount: NumberDecimal("0E-6176"), created: new Date(0), tags: [null, "中文"]}, + {_id: nativeOid(4), seq: NumberLong("4"), name: "delta", grp: "west", active: false, + ratio: 2.5, amount: NumberDecimal("99999.99"), created: ISODate("2038-01-19T03:14:07.000Z")} +]); +nativeFixture.createCollection("empty_records", {validator: {$jsonSchema: { + bsonType: "object", properties: {_id: {bsonType: "objectId"}, seq: {bsonType: "long"}} +}}}); +nativeFixture.createCollection("large_records", {validator: {$jsonSchema: { + bsonType: "object", properties: {_id: {bsonType: "objectId"}, seq: {bsonType: "long"}, payload: {bsonType: "string"}} +}}}); +nativeFixture.large_records.insertMany(Array.from({length: 2081}, (_, i) => ({ + _id: nativeOid(i + 1), seq: NumberLong(String(i)), payload: "payload_" + i % 7 +}))); +// The view removes _id and computes output fields. Reading its source would return different rows. +nativeFixture.createView("record_view", "typed_records", [ + {$match: {active: true}}, {$project: {_id: 0, seq: 1, label: {$concat: ["view-", "$name"]}}} +], {collation: {locale: "en", strength: 2}}); +nativeFixture.createCollection("measurements", {timeseries: {timeField: "observed", metaField: "device"}}); +nativeFixture.measurements.insertMany([ + {observed: ISODate("2024-01-01T00:00:00.001Z"), device: "a", value: NumberInt(1)}, + {observed: ISODate("2024-01-01T00:00:00.001Z"), device: "a", value: NumberInt(2)}, + {observed: ISODate("2024-01-02T00:00:00.000Z"), device: "b", value: NumberInt(3)} +]); +nativeFixture.special_records.insertOne({ + _id: nativeOid(1), integral: NumberLong("9007199254740993"), decimal: NumberDecimal("1.2300"), + date: new Date(-1), bytes: BinData(128, "AP+A"), pattern: /a.*b/im, + clock: Timestamp({t: 4294967295, i: 7}), low: MinKey(), high: MaxKey(), + code: Code("return n", {n: NumberInt(1)}), refs: [nativeOid(2), null], + "literal.dot": "literal", nested: {dot: "nested"}, "dollar$field": "dollar", "unicode": "中文" +}); +nativeFixture.mismatched.insertMany([ + {_id: nativeOid(1), seq: NumberLong("1"), n: NumberLong("10"), arr: [NumberInt(1), NumberInt(2)], + obj: {a: NumberInt(1), b: "ok"}, amount: NumberDecimal("1.2300"), blob: BinData(0, "AA==")}, + {_id: nativeOid(2), seq: NumberLong("2"), n: "bad", arr: [NumberInt(3), "bad"], + obj: {a: NumberInt(2), b: NumberInt(3)}, amount: NumberDecimal("1.234"), blob: BinData(128, "AA==")}, + {_id: nativeOid(3), seq: NumberLong("3"), n: NumberLong("30"), arr: [NumberInt(4)], + obj: {a: NumberInt(3)}, amount: NumberDecimal("2.00"), blob: BinData(0, "")} +]); +// A query-only validator cannot be represented as a complete column schema: auto must use document mode. +nativeFixture.createCollection("unsupported_validator", {validator: {value: {$gte: 0}}}); +nativeFixture.unsupported_validator.insertOne({_id: nativeOid(1), value: NumberInt(1)}); +// Existing documents may violate a validator introduced after insertion. +nativeFixture.historical_records.insertOne({seq: "historical-string"}); +nativeFixture.runCommand({collMod: "historical_records", validator: {$jsonSchema: { + bsonType: "object", properties: {seq: {bsonType: "long"}} +}}, validationLevel: "moderate"}); +nativeFixture.createRole({role: "nativeViewReader", privileges: [ + {resource: {db: "doris_mongodb_native", collection: "record_view"}, actions: ["find"]} +], roles: []}); +nativeFixture.createUser({user: "native_view_reader", pwd: "native_view_reader", + roles: [{role: "nativeViewReader", db: "doris_mongodb_native"}]}); + +// Explicit BSON null and absent keys must exercise every supported target family independently. +const boundaryFields = ["bool_v", "tiny_v", "small_v", "int_v", "long_v", "large_v", "float_v", "double_v", + "text_v", "oid_v", "bin_v", "dec32_v", "dec64_v", "dec128_v", "date_v", "date_precision_v", "timestamp_v", "regex_v", + "array_v", "object_v", "nested_v", "json_v", "decimal_json_v", "min_key_v", "max_key_v", "code_v", "scope_v"]; +const allNull = {_id: nativeOid(1), seq: NumberInt(1)}; +boundaryFields.forEach(name => allNull[name] = null); +nativeFixture.boundary_records.insertMany([ + allNull, {_id: nativeOid(2), seq: NumberInt(2)}, + {_id: nativeOid(3), seq: NumberInt(3), bool_v: false, + tiny_v: NumberInt(-128), small_v: NumberInt(-32768), int_v: NumberInt(-2147483648), + long_v: NumberLong("-9223372036854775808"), large_v: NumberLong("-9223372036854775808"), + float_v: Double("-3.4028234663852886e38"), double_v: Double("-1.7976931348623157e308"), + text_v: "", oid_v: ObjectId("000000000000000000000000"), bin_v: BinData(128, ""), + dec32_v: NumberDecimal("-9999999.99"), dec64_v: NumberDecimal("-99999999999999.9999"), + dec128_v: NumberDecimal("-9999999999999999999999999999999999"), + date_v: ISODate("0000-01-01T00:00:00.000Z"), date_precision_v: ISODate("0000-01-01T00:00:00Z"), + // MongoDB replaces a top-level zero Timestamp with server time; nesting preserves the boundary. + timestamp_v: {value: Timestamp({t: 0, i: 0})}, + regex_v: /^$/, array_v: [], object_v: {a: null, b: ""}, nested_v: [], json_v: []}, + {_id: nativeOid(4), seq: NumberInt(4), bool_v: true, + tiny_v: NumberInt(127), small_v: NumberInt(32767), int_v: NumberInt(2147483647), + long_v: NumberLong("9223372036854775807"), large_v: NumberLong("9223372036854775807"), + float_v: Double("3.4028234663852886e38"), double_v: Double("1.7976931348623157e308"), + text_v: "a\u0000中文😀", oid_v: ObjectId("ffffffffffffffffffffffff"), bin_v: BinData(128, "AP+Afw=="), + dec32_v: NumberDecimal("9999999.99"), dec64_v: NumberDecimal("99999999999999.9999"), + dec128_v: NumberDecimal("9999999999999999999999999999999999"), + date_v: ISODate("9999-12-31T23:59:59.999Z"), date_precision_v: ISODate("9999-12-31T23:59:59Z"), + timestamp_v: {value: Timestamp({t: 4294967295, i: 4294967295})}, + regex_v: /中文.*é/im, array_v: [null, NumberInt(-2147483648), NumberInt(2147483647)], + object_v: {a: NumberInt(2147483647), b: "中文"}, + nested_v: [null, {a: null, b: ""}, {a: NumberInt(2147483647), b: null}], json_v: {}, + min_key_v: MinKey(), max_key_v: MaxKey(), code_v: Code("return 1"), scope_v: Code("return n", {n: null})}, + {_id: nativeOid(5), seq: NumberInt(5), bool_v: false, tiny_v: NumberInt(0), small_v: NumberInt(0), + int_v: NumberInt(0), long_v: NumberLong("0"), large_v: NumberLong("0"), + float_v: Double("1.401298464324817e-45"), double_v: Double("5e-324"), bin_v: BinData(128, "AA=="), + text_v: "a".repeat(4096), dec32_v: NumberDecimal("-0.00"), dec64_v: NumberDecimal("0.0000"), + dec128_v: NumberDecimal("0E-6176"), date_v: new Date(-1), date_precision_v: new Date(0), + array_v: [null], object_v: {}, + nested_v: [null, {a: null, b: null}], json_v: ""}, + {_id: nativeOid(6), seq: NumberInt(6), float_v: Double("NaN"), double_v: Double("NaN"), + decimal_json_v: NumberDecimal("NaN")}, + {_id: nativeOid(7), seq: NumberInt(7), float_v: Double("Infinity"), double_v: Double("Infinity"), + decimal_json_v: NumberDecimal("9.999999999999999999999999999999999E+6144")}, + {_id: nativeOid(8), seq: NumberInt(8), float_v: Double("-Infinity"), double_v: Double("-Infinity"), + decimal_json_v: NumberDecimal("1E-6176")}, + {_id: nativeOid(9), seq: NumberInt(9), float_v: Double("-0.0"), double_v: Double("-0.0"), + decimal_json_v: NumberDecimal("-0.0000")}, + {_id: nativeOid(10), seq: NumberInt(10), float_v: Double("1.1754943508222875e-38"), + double_v: Double("2.2250738585072014e-308"), dec32_v: NumberDecimal("-0.01"), + dec64_v: NumberDecimal("0.0001"), dec128_v: NumberDecimal("0.0001"), date_v: new Date(0), + code_v: Code(""), scope_v: Code("", {})}, + {_id: nativeOid(11), seq: NumberInt(11), float_v: Double("0.0"), double_v: Double("0.0")} +]); + +// Values just beyond target ranges and values requiring coercion/rounding must fail atomically. +nativeFixture.boundary_errors.insertMany([ + {_id: nativeOid(1), seq: NumberInt(1), bool_v: NumberInt(1), tiny_v: NumberInt(128), + small_v: NumberInt(32768), int_v: NumberLong("2147483648"), long_v: Double("9007199254740992"), + large_v: "9223372036854775808", float_v: Double("0.1"), text_v: NumberInt(1), + oid_v: BinData(0, "AAAAAAAAAAAAAAAA"), bin_v: BinData(0, "AA=="), + dec32_v: NumberDecimal("10000000.00"), dec64_v: NumberDecimal("100000000000000.0000"), + dec128_v: NumberDecimal("1E+34"), date_v: new Date(253402300800000), date_precision_v: new Date(-1), + timestamp_v: NumberLong("1"), regex_v: "pattern", array_v: [NumberInt(1), "bad"], + object_v: {a: NumberInt(1), b: NumberInt(2)}, nested_v: [{a: NumberInt(1)}, {a: "bad"}]}, + {_id: nativeOid(2), seq: NumberInt(2), tiny_v: NumberInt(-129), small_v: NumberInt(-32769), + int_v: NumberLong("-2147483649"), float_v: Double("1e300"), + dec32_v: NumberDecimal("1.001"), dec64_v: NumberDecimal("0.00001"), + dec128_v: NumberDecimal("1E+6144"), date_v: new Date(-62167219200001), array_v: true, object_v: 1}, + {_id: nativeOid(3), seq: NumberInt(3), dec32_v: NumberDecimal("NaN"), + dec64_v: NumberDecimal("Infinity"), dec128_v: NumberDecimal("-Infinity")} +]); +// MongoDB permits this depth, while the native reader's declared decoding limit is lower. +let deepValue = NumberInt(1); +for (let i = 0; i < 65; ++i) deepValue = [deepValue]; +nativeFixture.deep_limit_records.insertOne({_id: nativeOid(1), value: deepValue}); + +// Legal validators outside the typed mapper's grammar must retain every value in document mode. +const fallbackSchemas = { + tuple_validator: {bsonType: "object", properties: {values: {bsonType: "array", + items: [{bsonType: "int"}, {bsonType: "string"}]}}}, + pattern_validator: {bsonType: "object", properties: {known: {bsonType: "int"}}, + patternProperties: {"^extra$": {bsonType: "string"}}}, + dependency_validator: {bsonType: "object", properties: {known: {bsonType: "int"}}, + dependencies: {known: {properties: {extra: {bsonType: "string"}}}}}, + nested_pattern_validator: {bsonType: "object", properties: {profile: {bsonType: "object", + properties: {known: {bsonType: "int"}}, patternProperties: {"^extra$": {bsonType: "string"}}}}} +}; +Object.entries(fallbackSchemas).forEach(([name, schema]) => { + nativeFixture.createCollection(name, {validator: {$jsonSchema: schema}}); + nativeFixture.getCollection(name).insertOne({_id: nativeOid(1), known: NumberInt(1), extra: "present", + values: [NumberInt(1), "x"], profile: {known: NumberInt(2), extra: "nested"}}); +}); +// Whole-document validation must use depth-bounded scratch even for a narrow scalar projection. +nativeFixture.wide_records.insertOne({_id: nativeOid(1), seq: NumberInt(1), + wide: Array.from({length: 900000}, () => ({}))}); +// Cross the driver's 128-name metadata batch without enlarging the normal test database. +const paginationFixture = db.getSiblingDB("doris_mongodb_pagination"); +for (let i = 0; i < 140; ++i) { + paginationFixture.createCollection("page_" + i.toString().padStart(3, "0")); +} diff --git a/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl b/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl index 07f5145cd6f70f..f091e9ce59b638 100644 --- a/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl +++ b/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl @@ -50,7 +50,17 @@ services: - >- const fixture = db.getSiblingDB('doris_mongodb_test'); quit(db.adminCommand('ping').ok && fixture.large_records.countDocuments() === 2081 - && fixture.getCollection('_schema').countDocuments({table: 'numeric_limits'}) === 1 ? 0 : 1); + && fixture.getCollection('_schema').countDocuments({table: 'numeric_limits'}) === 1 + && db.getSiblingDB('doris_mongodb_native').typed_records.countDocuments() === 4 + && db.getSiblingDB('doris_mongodb_native').large_records.countDocuments() === 2081 + && db.getSiblingDB('doris_mongodb_native').record_view.countDocuments() === 2 + && db.getSiblingDB('doris_mongodb_native').measurements.countDocuments() === 3 + && db.getSiblingDB('doris_mongodb_native').wide_records.countDocuments() === 1 + && db.getSiblingDB('doris_mongodb_native').tuple_validator.countDocuments() === 1 + && db.getSiblingDB('doris_mongodb_native').pattern_validator.countDocuments() === 1 + && db.getSiblingDB('doris_mongodb_native').dependency_validator.countDocuments() === 1 + && db.getSiblingDB('doris_mongodb_native').nested_pattern_validator.countDocuments() === 1 + && db.getSiblingDB('doris_mongodb_pagination').getCollectionNames().length === 140 ? 0 : 1); interval: 5s timeout: 5s retries: 30 diff --git a/fe/fe-connector/fe-connector-mongodb/README.md b/fe/fe-connector/fe-connector-mongodb/README.md index 47d7064cb95567..5edf1b2ca497cb 100644 --- a/fe/fe-connector/fe-connector-mongodb/README.md +++ b/fe/fe-connector/fe-connector-mongodb/README.md @@ -1,28 +1,138 @@ -# Native MongoDB client dependencies +# Native MongoDB Catalog -This module prepares the Java dependencies for the native MongoDB connector. It -bundles the official synchronous driver, its core, BSON, and BSON record codec -at one version, using a module-local BOM so other connector plugins keep their -own dependencies. The archive also carries the Snappy and Zstandard Java providers -so FE metadata reads accept the same compression URI options as the native BE. -The FE host supplies logging APIs and bindings. +The FE plugin uses MongoDB's synchronous Java driver for metadata. BE reads logical +collections directly with `libmongoc` and `libbson`; BSON values are converted by +Doris `DataTypeSerDe` implementations. No BE JVM or Trino connector is required. +The plugin archive is deployed to `plugins/connector/mongodb/` by `build.sh --fe`. -The dependency-only archive does not register a `ConnectorProvider` and cannot -create a MongoDB catalog. The FE build stages it as -`plugins/connector/mongodb.zip`, without unpacking it into an auto-loaded plugin -directory. A later connector implementation will add the provider and enable the -normal plugin deployment path. +```sql +CREATE CATALOG mongo_native PROPERTIES ( + "type" = "mongodb", + "mongodb.uri" = "mongodb://mongo.example:27017", + "mongodb.username" = "reader", + "mongodb.password" = "example-password", + "mongodb.auth_database" = "admin", + "mongodb.databases" = "sample", + "mongodb.schema.mode" = "auto" +); +``` + +The database allowlist is optional. It also permits discovery with accounts that +cannot list all databases. Reads require `find` on the logical collection or view. +Metadata discovery uses authorized, name-only listing. Validator schemas require +full `listCollections` metadata; explicit/document schemas also work with restricted +view-only readers. The pinned Java driver core operation returns a server-bound +metadata cursor with both authorization flags and object kinds; its public names +iterable exposes only strings. The driver owns continuation and cursor cleanup. +Views are queried by their own names, preserving their pipeline +and collation. Time-series collections return logical measurements, including +measurements without a unique `_id`; `system.buckets` is never exposed or read. + +## Schema policies + +- `auto` (default): explicit mapping for the namespace, otherwise an ordinary + collection's fully representable `$jsonSchema` validator, otherwise document mode. + It never samples rows or silently drops unrepresentable declared fields. +- `validator`: require a representable validator on an ordinary collection. +- `explicit`: require a catalog-owned mapping for each queried namespace. +- `document`: `_mongo_ejson` contains canonical Extended JSON and `_mongo_bson` + contains the original BSON bytes. Extended JSON preserves BSON type tags; it is + not ordinary JSON with automatic Doris numeric coercion. + +Validators constrain writes, not all historical documents. Mapped columns and +nested fields remain nullable. Tuple-array items, schema combinations, pattern-based +properties and schema-valued dependencies/additional properties require an explicit +mapping or whole-collection document fallback, including inside nested objects. +Missing fields and BSON null both become SQL NULL; +the original BSON preserves the distinction. Unsupported combinations use document +mode or produce an explicit error. Refresh metadata after changing a validator, +explicit mapping, view, or collection definition: + +```sql +REFRESH CATALOG mongo_native; +``` + +An explicit schema is a JSON object in `mongodb.schema`, keyed by `database.collection`: + +```json +{ + "sample.records": { + "columns": [ + {"name":"identifier","path":["_id"],"bson_types":["objectId"],"doris_type":"VARBINARY(12)"}, + {"name":"amount","path":["amount"],"bson_types":["decimal"],"doris_type":"DECIMAL(20,2)"}, + {"name":"literal","path":["a.b"],"doris_type":"STRING"}, + {"name":"nested","path":["a","b"],"doris_type":"STRING"}, + {"name":"payload","path":["payload"],"bson_types":["binData"],"subtype":0,"doris_type":"VARBINARY"} + ] + } +} +``` + +Paths contain exact case-sensitive segments. Doris column/struct names are folded +to lowercase. An optional `children` array overrides struct-field or array-element +bindings using the same shape. Struct fields are matched by name; arrays require +one element binding. Free-form objects and heterogeneous arrays can map to JSON. + +ObjectId maps to its raw 12 bytes: use `FROM_BINARY(identifier)` for hex display +and supported string comparisons, joins, or grouping. Direct VARBINARY operators +retain Doris's existing analysis restrictions. +The declared VARBINARY length is schema metadata; decoding preserves the full bytes, +matching the existing VARBINARY readers. +BinData mapped to VARBINARY requires an explicit subtype; arbitrary subtypes can be +preserved in Extended JSON. Timestamp maps to +`STRUCT`, preserving both unsigned 32-bit components. +BSON date maps to UTC `DATETIME(3)` or explicit BIGINT epoch milliseconds. Decimal128 +requires an explicit DECIMAL precision/scale for typed conversion; validator-derived +decimals use Extended JSON because they have no common fixed scale. + +`mongodb.type_mismatch` defaults to `error`. Choosing `null` replaces a failed top-level +field with SQL NULL after rolling back nested data; it does not suppress transport +failures or replace a malformed array element while keeping a partial array. Numeric +narrowing, decimal rounding, datetime precision loss, incompatible BSON tags/subtypes, +and values outside the target range are rejected. No implicit string coercion occurs. + +## Execution policy + +This implementation emits one split per logical namespace. The native cursor handles +find/getMore batches. Projection is materialized locally, and every SQL filter, +LIKE/REGEXP, join, aggregation, sort and LIMIT remains a Doris operation. Range +splitting, predicate/LIMIT pushdown and shared snapshots are separate work. + +`batch_size` and `query_timeout` are session variables. URI read preference settings +take precedence over the catalog mode, retaining ordered tag sets and maximum staleness +in both drivers. Without URI routing settings, `mongodb.read_preference` supplies the mode. +Catalog connection defaults: + +| Property | Default | +|---|---| +| `mongodb.read_preference` | `primary` | +| `mongodb.read_concern` | `majority` (`local` and `available` also supported) | +| `mongodb.connect_timeout_ms` | `10000` | +| `mongodb.server_selection_timeout_ms` | `10000` | +| `mongodb.socket_timeout_ms` | `1000` | +| `mongodb.pool_max_clients` | `16` | +| `mongodb.datetime_timezone` | `UTC` | + +A majority read does not imply a point-in-time snapshot. Explicit snapshot/time-travel +requests are unsupported. EOF, early termination, cancellation and errors close the +cursor and return its exclusive client lease. Pool identities include catalog +configuration and credentials. Driver errors expose stable error codes without +source values or connection strings. Keep network timeouts small enough for the +required cancellation responsiveness. + +## Dependencies and tests -The BE uses the MongoDB C Driver (`libmongoc` and `libbson`) built together by: +Java dependencies use a module-local MongoDB BOM; the FE host supplies SPI and logging. +Build the matching native libraries with: ```sh bash thirdparty/build-thirdparty.sh mongo_c_driver ``` -The native build produces PIC static archives with OpenSSL, Cyrus SASL, SRV DNS, -Snappy, zstd and zlib support. It reuses Doris-managed dependencies and the driver's -bundled utf8proc for SCRAM-SHA-256. It does not require the C++ driver, a BE JVM, -Reactive Streams, or client-side encryption libraries. The reader and BSON -`DataTypeSerDe` implementation are separate changes. The BE CMake targets `bson2` -and `mongoc2` expose the static headers, definitions and dependency closure; they -are not added to the common BE link set until a reader consumes them. +FE unit tests reside in this module, BSON SerDe unit tests in +`be/test/core/data_type_serde/data_type_serde_bson_test.cpp`, and reader lifecycle tests +in `be/test/format_v2/table/mongodb_reader_test.cpp`. The regression suite +`external_table_p0/mongodb/test_mongodb_native.groovy` uses the disposable MongoDB +fixture under `docker/thirdparties/docker-compose/mongodb/`, enabled with +`enableMongoDBTest`, `externalEnvIp`, `mongodb_port`, `mongodb_user`, and +`mongodb_password` regression settings. diff --git a/fe/fe-connector/fe-connector-mongodb/pom.xml b/fe/fe-connector/fe-connector-mongodb/pom.xml index d98638186a11e2..1915bdf3503892 100644 --- a/fe/fe-connector/fe-connector-mongodb/pom.xml +++ b/fe/fe-connector/fe-connector-mongodb/pom.xml @@ -29,7 +29,7 @@ under the License. fe-connector-mongodb jar - Doris FE Connector - MongoDB Dependencies + Doris FE Connector - MongoDB Client dependency bundle for the native MongoDB connector. 5.14.0 @@ -50,6 +50,27 @@ under the License. + + ${project.groupId} + fe-connector-spi + ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + + ${project.groupId} + fe-thrift + ${project.version} + provided + + + org.junit.jupiter + junit-jupiter + test + org.mongodb mongodb-driver-sync diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/assembly/plugin-zip.xml b/fe/fe-connector/fe-connector-mongodb/src/main/assembly/plugin-zip.xml index 9fc745f112632d..b5e54355da1f82 100644 --- a/fe/fe-connector/fe-connector-mongodb/src/main/assembly/plugin-zip.xml +++ b/fe/fe-connector/fe-connector-mongodb/src/main/assembly/plugin-zip.xml @@ -42,6 +42,7 @@ under the License. org.apache.logging.log4j:* org.slf4j:* + org.apache.doris:* diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogClient.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogClient.java new file mode 100644 index 00000000000000..cc472dba5009ae --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogClient.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.bson.Document; + +import java.util.List; +import java.util.Optional; + +/** Metadata transport boundary, allowing tests to supply metadata without mocking driver internals. */ +interface MongoCatalogClient extends AutoCloseable { + List databases(); + + List collections(String database); + + default Optional collection(String database, String name) { + return collections(database).stream().filter(d -> name.equals(d.getString("name"))).findFirst(); + } + + @Override + void close(); +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogProperties.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogProperties.java new file mode 100644 index 00000000000000..6421184506587b --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoCatalogProperties.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoCredential; +import com.mongodb.ReadConcern; +import com.mongodb.ReadConcernLevel; +import com.mongodb.ReadPreference; +import org.bson.Document; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** Immutable catalog configuration; query policies and driver objects are never stored in table properties. */ +public final class MongoCatalogProperties { + @ConnectorProperty(names = {"mongodb.uri"}, sensitive = true, description = "MongoDB connection URI") + private String uri; + @ConnectorProperty(names = {"mongodb.username"}, required = false, description = "MongoDB user") + private String username = ""; + @ConnectorProperty(names = {"mongodb.password"}, required = false, sensitive = true, + description = "MongoDB password") + private String password = ""; + @ConnectorProperty(names = {"mongodb.auth_database"}, required = false, description = "Authentication database") + private String authDatabase = "admin"; + @ConnectorProperty(names = {"mongodb.databases"}, required = false, description = "Database allowlist") + private String databases = ""; + @ConnectorProperty(names = {"mongodb.schema.mode"}, required = false, description = "Schema source policy") + private String schemaMode = "auto"; + @ConnectorProperty(names = {"mongodb.schema"}, required = false, description = "Explicit schema manifest") + private String schema = "{}"; + @ConnectorProperty(names = {"mongodb.type_mismatch"}, required = false, description = "error or null") + private String typeMismatch = "error"; + @ConnectorProperty(names = {"mongodb.datetime_timezone"}, required = false, + description = "BSON date interpretation") + private String datetimeTimezone = "UTC"; + @ConnectorProperty(names = {"mongodb.read_preference"}, required = false, description = "Default read routing") + private String readPreference = "primary"; + @ConnectorProperty(names = {"mongodb.read_concern"}, required = false, description = "Default read concern") + private String readConcern = "majority"; + @ConnectorProperty(names = {"mongodb.connect_timeout_ms"}, required = false, description = "Connection timeout") + private int connectTimeoutMs = 10000; + @ConnectorProperty(names = {"mongodb.server_selection_timeout_ms"}, required = false, + description = "Server selection timeout") + private int serverSelectionTimeoutMs = 10000; + @ConnectorProperty(names = {"mongodb.socket_timeout_ms"}, required = false, description = "Socket timeout") + private int socketTimeoutMs = 1000; + @ConnectorProperty(names = {"mongodb.pool_max_clients"}, required = false, description = "Per-catalog client bound") + private int poolMaxClients = 16; + + private ConnectionString connectionString; + + private MongoCatalogProperties() { + } + + public static MongoCatalogProperties of(Map properties) { + Set known = Set.of("mongodb.uri", "mongodb.username", "mongodb.password", "mongodb.auth_database", + "mongodb.databases", "mongodb.schema.mode", "mongodb.schema", "mongodb.type_mismatch", + "mongodb.datetime_timezone", "mongodb.read_preference", "mongodb.read_concern", + "mongodb.connect_timeout_ms", "mongodb.server_selection_timeout_ms", "mongodb.socket_timeout_ms", + "mongodb.pool_max_clients"); + for (String key : properties.keySet()) { + if (key.startsWith("mongodb.") && !known.contains(key)) { + throw new IllegalArgumentException("Unknown MongoDB catalog property: " + key); + } + } + MongoCatalogProperties p = new MongoCatalogProperties(); + ConnectorPropertiesUtils.bindConnectorProperties(p, properties); + try { + p.connectionString = new ConnectionString(p.uri); + } catch (RuntimeException e) { + // Driver parse exceptions may echo a credential-bearing URI; never retain that cause. + throw new IllegalArgumentException("Invalid mongodb.uri"); + } + if (p.connectionString.getCredential() != null && (!p.username.isEmpty() || !p.password.isEmpty())) { + throw new IllegalArgumentException("MongoDB credentials must use one configuration source"); + } + if (p.username.isEmpty() && !p.password.isEmpty()) { + throw new IllegalArgumentException("mongodb.password requires mongodb.username"); + } + if (Boolean.TRUE.equals(p.connectionString.getSslInvalidHostnameAllowed())) { + throw new IllegalArgumentException("MongoDB TLS hostname validation must remain enabled"); + } + p.schemaMode = p.schemaMode.toLowerCase(Locale.ROOT); + require(Set.of("auto", "validator", "explicit", "document").contains(p.schemaMode), "mongodb.schema.mode"); + require(Set.of("error", "null").contains(p.typeMismatch), "mongodb.type_mismatch"); + require("UTC".equals(p.datetimeTimezone), "mongodb.datetime_timezone"); + require(Set.of("local", "majority", "available").contains(p.readConcern), "mongodb.read_concern"); + require(p.connectTimeoutMs > 0 && p.connectTimeoutMs <= 300000, "mongodb.connect_timeout_ms"); + require(p.serverSelectionTimeoutMs > 0 && p.serverSelectionTimeoutMs <= 300000, + "mongodb.server_selection_timeout_ms"); + require(p.socketTimeoutMs > 0 && p.socketTimeoutMs <= 300000, "mongodb.socket_timeout_ms"); + require(p.poolMaxClients > 0 && p.poolMaxClients <= 1024, "mongodb.pool_max_clients"); + require(p.schema.length() <= 1048576, "mongodb.schema size"); + try { + p.readPreference = ReadPreference.valueOf(p.readPreference).getName(); + Document manifest = Document.parse(p.schema); + require(manifest.size() <= 1024, "mongodb.schema table count"); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid MongoDB read preference or schema manifest"); + } + if (p.connectionString.getReadConcern() != null) { + String level = p.connectionString.getReadConcern().getLevel().getValue(); + require(Set.of("local", "majority", "available").contains(level), "URI readConcernLevel"); + p.readConcern = level; + } + if (p.connectionString.getReadPreference() != null) { + p.readPreference = p.connectionString.getReadPreference().getName(); + } + return p; + } + + private static void require(boolean condition, String property) { + if (!condition) { + throw new IllegalArgumentException("Invalid or unsupported " + property); + } + } + + MongoClientSettings clientSettings() { + MongoClientSettings.Builder builder = MongoClientSettings.builder().applyConnectionString(connectionString) + .applyToSocketSettings(s -> s.connectTimeout(connectTimeoutMs, TimeUnit.MILLISECONDS) + .readTimeout(socketTimeoutMs, TimeUnit.MILLISECONDS)) + .applyToClusterSettings(s -> s.serverSelectionTimeout(serverSelectionTimeoutMs, TimeUnit.MILLISECONDS)) + .applyToConnectionPoolSettings(s -> s.maxSize(poolMaxClients)) + // getName() is only the scan descriptor's mode; rebuilding from it drops URI + // tag sets, max staleness and hedge options. URI routing keeps its precedence. + .readPreference(connectionString.getReadPreference() != null + ? connectionString.getReadPreference() : ReadPreference.valueOf(readPreference)) + .readConcern(new ReadConcern(ReadConcernLevel.fromString(readConcern))); + if (!username.isEmpty()) { + builder.credential(MongoCredential.createCredential(username, authDatabase, password.toCharArray())); + } + return builder.build(); + } + + public String getUri() { + return uri; + } + + String getUsername() { + return username; + } + + String getPassword() { + return password; + } + + String getAuthDatabase() { + return authDatabase; + } + + String getSchemaMode() { + return schemaMode; + } + + String getSchema() { + return schema; + } + + String getTypeMismatch() { + return typeMismatch; + } + + String getReadPreference() { + return readPreference; + } + + String getReadConcern() { + return readConcern; + } + + int getConnectTimeoutMs() { + return connectTimeoutMs; + } + + int getServerSelectionTimeoutMs() { + return serverSelectionTimeoutMs; + } + + int getSocketTimeoutMs() { + return socketTimeoutMs; + } + + int getPoolMaxClients() { + return poolMaxClients; + } + + List getDatabases() { + return Arrays.stream(databases.split(",")).map(String::trim).filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + String configurationVersion() { + try { + String value = new Document("uri", uri).append("username", username).append("password", password) + .append("auth_database", authDatabase).append("pool", poolMaxClients) + .append("connect", connectTimeoutMs).append("selection", serverSelectionTimeoutMs) + .append("socket", socketTimeoutMs).toJson(); + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is required", e); + } + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoColumnHandle.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoColumnHandle.java new file mode 100644 index 00000000000000..378ae96d5a7a94 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoColumnHandle.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; + +import java.util.Objects; + +/** Logical BSON tags and paths survive planning even when several BSON types share a Doris type. */ +final class MongoColumnHandle implements ConnectorColumnHandle { + private static final long serialVersionUID = 1L; + private final String name; + private final String schemaVersion; + private final String bindingJson; + + MongoColumnHandle(MongoSchema.Binding binding, String schemaVersion) { + this.name = binding.getName(); + this.schemaVersion = schemaVersion; + this.bindingJson = binding.toDocument().toJson(); + } + + String getSchemaVersion() { + return schemaVersion; + } + + String getBindingJson() { + return bindingJson; + } + + @Override + public int hashCode() { + return Objects.hash(name, schemaVersion, bindingJson); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof MongoColumnHandle)) { + return false; + } + MongoColumnHandle column = (MongoColumnHandle) other; + return name.equals(column.name) && schemaVersion.equals(column.schemaVersion) + && bindingJson.equals(column.bindingJson); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnector.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnector.java new file mode 100644 index 00000000000000..098a8736c7ad8b --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnector.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; + +final class MongoConnector implements Connector { + private final MongoCatalogProperties properties; + private MongoCatalogClient client; + private boolean closed; + + MongoConnector(MongoCatalogProperties properties) { + this.properties = properties; + } + + @Override + public synchronized ConnectorMetadata getMetadata(ConnectorSession session) { + if (closed) { + throw new IllegalStateException("MongoDB catalog is closed"); + } + if (client == null) { + client = new MongoDriverCatalogClient(properties); + } + return new MongoConnectorMetadata(client, properties); + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return new MongoScanPlanProvider(properties); + } + + @Override + public synchronized void close() { + closed = true; + if (client != null) { + client.close(); + client = null; + } + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorMetadata.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorMetadata.java new file mode 100644 index 00000000000000..d224b172989018 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorMetadata.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScopes; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.bson.Document; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +final class MongoConnectorMetadata implements ConnectorMetadata { + static final String TABLE_NAMESPACE = "mongodb.table"; + private final MongoCatalogClient client; + private final MongoCatalogProperties properties; + + MongoConnectorMetadata(MongoCatalogClient client, MongoCatalogProperties properties) { + this.client = client; + this.properties = properties; + } + + @Override + public List listDatabaseNames(ConnectorSession session) { + return client.databases(); + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return listDatabaseNames(session).contains(dbName); + } + + @Override + public List listTableNames(ConnectorSession session, String dbName) { + if (!allowedDatabase(dbName)) { + return List.of(); + } + List names = new ArrayList<>(); + for (Document object : client.collections(dbName)) { + if (visible(object)) { + names.add(object.getString("name")); + } + } + return names; + } + + @Override + public Optional getTableHandle(ConnectorSession session, String dbName, String tableName) { + if (!allowedDatabase(dbName) || tableName.startsWith("system.")) { + return Optional.empty(); + } + return ConnectorStatementScopes.resolveInStatement(session, TABLE_NAMESPACE, dbName, tableName, + () -> client.collection(dbName, tableName).filter(MongoConnectorMetadata::visible) + .map(metadata -> new MongoTableHandle(dbName, metadata, properties))); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + MongoTableHandle table = (MongoTableHandle) handle; + return new ConnectorTableSchema(table.getName(), table.getSchema().getColumns(), "MongoDB", + Map.of("mongodb.object_kind", table.getObjectKind(), + "mongodb.schema_source", table.getSchema().getSource(), + "mongodb.schema_version", table.getSchema().getVersion())); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle handle) { + Map columns = new LinkedHashMap<>(); + MongoSchema schema = ((MongoTableHandle) handle).getSchema(); + for (MongoSchema.Binding column : schema.getBindings()) { + columns.put(column.getName(), new MongoColumnHandle(column, schema.getVersion())); + } + return columns; + } + + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, long tableId, String tableName, + String dbName, String remoteName, int numCols, long catalogId) { + // The generic file scan path expects a Hive-shaped descriptor, without source credentials. + TTableDescriptor descriptor = new TTableDescriptor(tableId, TTableType.HIVE_TABLE, numCols, 0, + tableName, dbName); + descriptor.setHiveTable(new THiveTable(dbName, remoteName, new HashMap<>())); + return descriptor; + } + + private boolean allowedDatabase(String database) { + return properties.getDatabases().isEmpty() || properties.getDatabases().contains(database); + } + + private static boolean visible(Document metadata) { + String name = metadata.getString("name"); + String kind = metadata.getString("type"); + return name != null && !name.startsWith("system.") + && Set.of("collection", "view", "timeseries").contains(kind); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorProvider.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorProvider.java new file mode 100644 index 00000000000000..e654b415df5fcf --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoConnectorProvider.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; +import org.apache.doris.connector.spi.ConnectorSecuritySurface; + +import java.util.Map; + +public final class MongoConnectorProvider implements ConnectorProvider { + @Override + public String getType() { + return "mongodb"; + } + + @Override + public void validateProperties(Map properties) { + MongoCatalogProperties.of(properties); + } + + @Override + public ConnectorSecuritySurface securitySurface() { + return ConnectorSecuritySurface.ofSsrfEndpoints("mongodb.uri"); + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + return new MongoConnector(MongoCatalogProperties.of(properties)); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClient.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClient.java new file mode 100644 index 00000000000000..c30b7b91be0377 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClient.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import com.mongodb.MongoCommandException; +import com.mongodb.MongoException; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.internal.MongoClientImpl; +import com.mongodb.internal.operation.BatchCursor; +import com.mongodb.internal.operation.ListCollectionsOperation; +import org.bson.Document; +import org.bson.codecs.DocumentCodec; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +/** Catalog-owned thread-safe driver; every metadata cursor/session is local and deterministically closed. */ +final class MongoDriverCatalogClient implements MongoCatalogClient { + private final MongoClient client; + private final MongoCatalogProperties properties; + private final Function, BatchCursor> metadataCursor; + + MongoDriverCatalogClient(MongoCatalogProperties properties) { + this(MongoClients.create(properties.clientSettings()), properties); + } + + MongoDriverCatalogClient(MongoClient client, MongoCatalogProperties properties) { + this(client, properties, operation -> { + // Driver 5.14 exposes authorized name-only results as strings in its public iterable. + // Use its pinned core operation to retain kind metadata AND the server-bound cursor. + MongoClientImpl driver = (MongoClientImpl) client; + return driver.getOperationExecutor().withTimeoutSettings(driver.getTimeoutSettings() + .withMaxTimeMS(properties.getServerSelectionTimeoutMs())) + .execute(operation, ReadPreference.primary(), ReadConcern.DEFAULT); + }); + } + + MongoDriverCatalogClient(MongoClient client, MongoCatalogProperties properties, + Function, BatchCursor> metadataCursor) { + this.properties = properties; + this.client = client; + this.metadataCursor = metadataCursor; + } + + @Override + public List databases() { + if (!properties.getDatabases().isEmpty()) { + return List.copyOf(properties.getDatabases()); + } + try (MongoCursor cursor = client.listDatabaseNames().iterator()) { + List names = new ArrayList<>(); + cursor.forEachRemaining(names::add); + return names; + } catch (MongoException e) { + throw failure(e); + } + } + + @Override + public List collections(String database) { + return names(database, null); + } + + @Override + public Optional collection(String database, String name) { + try (MongoCursor cursor = metadataDatabase(database).listCollections() + .filter(new Document("name", name)).batchSize(1) + .maxTime(properties.getServerSelectionTimeoutMs(), TimeUnit.MILLISECONDS).iterator()) { + return cursor.hasNext() ? Optional.of(cursor.next()) : Optional.empty(); + } catch (MongoCommandException e) { + if (e.getErrorCode() == 13 && !"validator".equals(properties.getSchemaMode())) { + // A view-only reader may enumerate authorized names but cannot read options/UUIDs. + // Auto must fall back to document mode when validator metadata is inaccessible. + // Preserve the returned kind and absence of identity; never inspect its base collection. + return names(database, name).stream().findFirst(); + } + throw failure(e); + } catch (MongoException e) { + throw failure(e); + } + } + + private MongoDatabase metadataDatabase(String database) { + // listCollections is metadata, not a data read; it cannot inherit a data-scan snapshot policy. + return client.getDatabase(database).withReadConcern(ReadConcern.DEFAULT) + .withReadPreference(ReadPreference.primary()); + } + + private List names(String database, String name) { + ListCollectionsOperation operation = + new ListCollectionsOperation<>(database, new DocumentCodec(), null) + .nameOnly(true).authorizedCollections(true).batchSize(128); + if (name != null) { + operation.filter(new Document("name", name).toBsonDocument()); + } + // An ordinary ClientSession does not pin a mongos. BatchCursor retains the original + // connection source for getMore and killCursors, including failure and early-close paths. + try (BatchCursor cursor = metadataCursor.apply(operation)) { + List result = new ArrayList<>(); + while (cursor.hasNext()) { + for (Document object : cursor.next()) { + result.add(new Document(object).append("metadata_available", false)); + if (result.size() > 100000) { + throw new DorisConnectorException("MongoDB database exceeds metadata listing limit"); + } + } + } + return result; + } catch (MongoException e) { + throw failure(e); + } + } + + private static DorisConnectorException failure(MongoException error) { + // Source messages can contain document values or URI credentials. Only expose the stable code. + return new DorisConnectorException("MongoDB metadata request failed (code " + error.getCode() + ")"); + } + + @Override + public void close() { + client.close(); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanPlanProvider.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanPlanProvider.java new file mode 100644 index 00000000000000..3aa9ecb2b0ff21 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanPlanProvider.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.pushdown.ConnectorExpression; +import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.thrift.TMongoScanDesc; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +final class MongoScanPlanProvider implements ConnectorScanPlanProvider { + private final MongoCatalogProperties properties; + + MongoScanPlanProvider(MongoCatalogProperties properties) { + this.properties = properties; + } + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + MongoTableHandle handle = (MongoTableHandle) request.getTableHandle(); + for (ConnectorColumnHandle column : request.getColumns()) { + if (!(column instanceof MongoColumnHandle) + || !handle.getSchema().getVersion().equals(((MongoColumnHandle) column).getSchemaVersion())) { + throw new IllegalArgumentException("MongoDB column schema changed; refresh metadata and retry"); + } + } + TMongoScanDesc d = new TMongoScanDesc(); + d.setVersion(1); + d.setCatalogId(session.getCatalogId()); + d.setConfigurationVersion(properties.configurationVersion()); + d.setDatabase(handle.getDatabase()); + d.setCollection(handle.getName()); + d.setObjectKind(handle.getObjectKind()); + d.setCollectionUuid(handle.getUuid()); + d.setSchemaVersion(handle.getSchema().getVersion()); + d.setBindingsJson(handle.getSchema().toBindingsJson()); + d.setUri(properties.getUri()); + d.setUsername(properties.getUsername()); + d.setPassword(properties.getPassword()); + d.setAuthDatabase(properties.getAuthDatabase()); + d.setReadPreference(properties.getReadPreference()); + d.setReadConcern(properties.getReadConcern()); + d.setTypeMismatchAsNull("null".equals(properties.getTypeMismatch())); + Map policies = session.getSessionProperties(); + d.setBatchSize(bounded(policies, "batch_size", 1024, 1, 65536)); + d.setTimeoutMs(1000L * bounded(policies, "query_timeout", 300, 1, Integer.MAX_VALUE / 1000)); + d.setConnectTimeoutMs(properties.getConnectTimeoutMs()); + d.setServerSelectionTimeoutMs(properties.getServerSelectionTimeoutMs()); + d.setSocketTimeoutMs(properties.getSocketTimeoutMs()); + d.setPoolMaxClients(properties.getPoolMaxClients()); + // One cursor owns the entire logical namespace. Filters and LIMIT remain Doris residuals. + return List.of(new MongoScanRange(d)); + } + + private static int bounded(Map properties, String name, int fallback, int min, int max) { + try { + int value = Integer.parseInt(properties.getOrDefault(name, Integer.toString(fallback))); + if (value >= min && value <= max) { + return value; + } + } catch (NumberFormatException ignored) { + // Reject malformed query policy rather than silently changing execution limits. + } + throw new IllegalArgumentException("Invalid MongoDB query policy: " + name); + } + + @Override + public Map getScanNodeProperties(ConnectorSession session, ConnectorTableHandle handle, + List columns, Optional filter) { + MongoTableHandle table = (MongoTableHandle) handle; + return Map.of("object_kind", table.getObjectKind(), "schema_source", table.getSchema().getSource(), + "schema_version", table.getSchema().getVersion()); + } + + @Override + public void appendExplainInfo(StringBuilder output, String prefix, Map nodeProperties) { + output.append(prefix).append("MongoDB object: ").append(nodeProperties.get("object_kind")) + .append("; schema: ").append(nodeProperties.get("schema_source")) + .append("; version: ").append(nodeProperties.get("schema_version")).append("\n"); + } + +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanRange.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanRange.java new file mode 100644 index 00000000000000..8a2f827a5e7573 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoScanRange.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TMongoScanDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.Map; +import java.util.Optional; + +final class MongoScanRange implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + private final TMongoScanDesc descriptor; + + MongoScanRange(TMongoScanDesc descriptor) { + this.descriptor = descriptor.deepCopy(); + } + + @Override + public Optional getPath() { + return Optional.of("/dummyPath"); + } + + @Override + public String getSplitIdentity() { + return descriptor.getDatabase() + "." + descriptor.getCollection() + ":" + descriptor.getSchemaVersion(); + } + + @Override + public String getFileFormat() { + return "mongodb"; + } + + @Override + public String getTableFormatType() { + return "mongodb"; + } + + @Override + public boolean isNativeReadRange() { + return true; + } + + @Override + public Map getProperties() { + // Generic scan properties appear in EXPLAIN; credentials belong only to the typed RPC descriptor. + return Map.of(); + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setMongodbParams(descriptor.deepCopy()); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoSchema.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoSchema.java new file mode 100644 index 00000000000000..77dd8dfc57a3f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoSchema.java @@ -0,0 +1,536 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorType; + +import org.bson.Document; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** A statement-pinned mapping. Validator constraints describe writes, never historical type guarantees. */ +final class MongoSchema { + static final int MAX_DEPTH = 64; + static final int MAX_FIELDS = 4096; + private static final Map BSON_TYPES = Map.ofEntries( + Map.entry("double", 1), Map.entry("string", 2), Map.entry("object", 3), Map.entry("array", 4), + Map.entry("binData", 5), Map.entry("undefined", 6), Map.entry("objectId", 7), Map.entry("bool", 8), + Map.entry("date", 9), Map.entry("null", 10), Map.entry("regex", 11), Map.entry("dbPointer", 12), + Map.entry("javascript", 13), Map.entry("symbol", 14), Map.entry("javascriptWithScope", 15), + Map.entry("int", 16), Map.entry("timestamp", 17), Map.entry("long", 18), Map.entry("decimal", 19), + Map.entry("minKey", 255), Map.entry("maxKey", 127)); + + static final class Binding { + private final String name; + private final List path; + private final ConnectorType type; + private final List bsonTypes; + private final List children; + private final int subtype; + private final String special; + + Binding(String name, List path, ConnectorType type, List bsonTypes, + List children, int subtype, String special) { + this.name = name.toLowerCase(Locale.ROOT); + this.path = List.copyOf(path); + this.type = type; + this.bsonTypes = List.copyOf(bsonTypes); + this.children = List.copyOf(children); + this.subtype = subtype; + this.special = special; + } + + String getName() { + return name; + } + + List getPath() { + return path; + } + + Document toDocument() { + return new Document("name", name).append("path", path).append("bson_types", bsonTypes) + .append("children", children.stream().map(Binding::toDocument).collect(Collectors.toList())) + .append("subtype", subtype).append("special", special) + .append("doris_type", type.getTypeName()).append("precision", type.getPrecision()) + .append("scale", type.getScale()); + } + + ConnectorColumn column() { + // Validator requirements do not establish a key, and views/time-series may have no unique _id. + return new ConnectorColumn(name, type, "MongoDB BSON types=" + bsonTypes + + (subtype < 0 ? "" : ", subtype=" + subtype), true, null, false); + } + } + + private final List bindings; + private final String source; + private final String version; + private final String fallbackReason; + + private MongoSchema(List bindings, String source, String identity, String fallbackReason) { + this(bindings, source, identity, fallbackReason, false); + } + + private MongoSchema(List bindings, String source, String identity, String fallbackReason, + boolean pinnedVersion) { + this.bindings = List.copyOf(bindings); + this.source = source; + this.version = pinnedVersion ? identity : digest(identity + source + toBindingsJson()); + this.fallbackReason = fallbackReason; + } + + String toPinnedJson() { + return new Document("source", source).append("schema_version", version).append("reason", fallbackReason) + .append("bindings", Document.parse(toBindingsJson())).toJson(); + } + + static MongoSchema fromPinnedJson(String json) { + Document d = Document.parse(json); + List bindings = new ArrayList<>(); + for (Document value : d.get("bindings", Document.class).getList("columns", Document.class)) { + bindings.add(pinnedBinding(value)); + } + return new MongoSchema(bindings, d.getString("source"), d.getString("schema_version"), + d.getString("reason"), true); + } + + private static Binding pinnedBinding(Document value) { + List children = new ArrayList<>(); + for (Document child : value.getList("children", Document.class)) { + children.add(pinnedBinding(child)); + } + String typeName = value.getString("doris_type"); + ConnectorType type = new ConnectorType(typeName, value.getInteger("precision"), value.getInteger("scale"), + children.stream().map(b -> b.type).collect(Collectors.toList()), + "STRUCT".equals(typeName) + ? children.stream().map(b -> b.name).collect(Collectors.toList()) : List.of()); + return new Binding(value.getString("name"), value.getList("path", String.class), type, + value.getList("bson_types", Integer.class), children, value.getInteger("subtype"), + value.getString("special")); + } + + static MongoSchema resolve(String namespace, Document metadata, MongoCatalogProperties properties) { + String mode = properties.getSchemaMode(); + Document explicit = Document.parse(properties.getSchema()).get(namespace, Document.class); + String identity = namespace + metadata.toJson() + properties.getSchema(); + if ("explicit".equals(mode) || ("auto".equals(mode) && explicit != null)) { + if (explicit == null) { + throw new IllegalArgumentException("No explicit MongoDB schema for the requested namespace"); + } + List definitions = explicit.getList("columns", Document.class); + if (definitions == null || definitions.isEmpty() || definitions.size() > MAX_FIELDS) { + throw new IllegalArgumentException("MongoDB explicit schema requires bounded nonempty columns"); + } + List columns = new ArrayList<>(); + int[] count = {0}; + for (Document definition : definitions) { + columns.add(explicitBinding(definition, 0, count)); + } + unique(columns); + return new MongoSchema(columns, "DORIS_EXPLICIT", identity, ""); + } + if ("document".equals(mode)) { + return document(identity, ""); + } + Document options = metadata.get("options", new Document()); + Document validator = options.get("validator", new Document()); + Document jsonSchema = validator.get("$jsonSchema", Document.class); + if (!"collection".equals(metadata.getString("type")) || jsonSchema == null || validator.size() != 1) { + if ("validator".equals(mode)) { + throw new IllegalArgumentException( + "MongoDB validator mode requires an ordinary collection JSON schema"); + } + return document(identity, "No applicable JSON schema validator"); + } + try { + List fields = objectFields(jsonSchema, 0, new int[] {0}); + if (fields.isEmpty()) { + throw new IllegalArgumentException("Validator has no declared fields"); + } + return new MongoSchema(fields, "MONGODB_VALIDATOR", identity, ""); + } catch (IllegalArgumentException e) { + if ("validator".equals(mode)) { + throw e; + } + // A partial mapping hides unsupported fields. Auto falls back for the entire collection. + return document(identity, "Validator cannot be mapped completely"); + } + } + + private static MongoSchema document(String identity, String reason) { + return new MongoSchema(List.of( + new Binding("_mongo_ejson", List.of(), ConnectorType.of("JSONB"), List.of(), + List.of(), -1, "ejson"), + new Binding("_mongo_bson", List.of(), ConnectorType.of("VARBINARY"), List.of(), + List.of(), -1, "bson")), "DOCUMENT", identity, reason); + } + + private static void requireRepresentableValidator(Document schema) { + // For example, properties:{known:...} plus patternProperties:{"^extra$":...} must + // not publish only known. Apply this check recursively before accepting any typed fields. + Document patterns = schema.get("patternProperties", Document.class); + Document dependencies = schema.get("dependencies", Document.class); + if (schema.containsKey("anyOf") || schema.containsKey("oneOf") || schema.containsKey("allOf") + || schema.containsKey("not") || (patterns != null && !patterns.isEmpty()) + || schema.get("additionalProperties") instanceof Document + || (dependencies != null && dependencies.values().stream().anyMatch(v -> v instanceof Document))) { + throw new IllegalArgumentException("MongoDB schema constructs require an explicit mapping"); + } + } + + private static List objectFields(Document schema, int depth, int[] count) { + requireRepresentableValidator(schema); + Document properties = schema.get("properties", Document.class); + if (properties == null) { + return List.of(); + } + List fields = new ArrayList<>(); + for (Map.Entry field : properties.entrySet()) { + if (!(field.getValue() instanceof Document)) { + throw new IllegalArgumentException("Invalid MongoDB field schema"); + } + fields.add(validatorBinding(field.getKey(), (Document) field.getValue(), depth + 1, count)); + } + unique(fields); + return fields; + } + + private static Binding validatorBinding(String name, Document definition, int depth, int[] count) { + bounded(depth, count); + requireRepresentableValidator(definition); + Object declared = definition.get("bsonType"); + List names = new ArrayList<>(); + if (declared instanceof String) { + names.add((String) declared); + } else if (declared instanceof List) { + for (Object value : (List) declared) { + if (!(value instanceof String)) { + throw new IllegalArgumentException("Invalid BSON type union"); + } + names.add((String) value); + } + } else { + throw new IllegalArgumentException("MongoDB field requires bsonType"); + } + List tags = tags(names); + Set nonNull = new HashSet<>(names); + nonNull.remove("null"); + List children = List.of(); + ConnectorType type; + if (nonNull.equals(Set.of("int", "long"))) { + type = ConnectorType.of("BIGINT"); + } else if (nonNull.size() > 1) { + throw new IllegalArgumentException("Mixed BSON types require an explicit or document schema"); + } else { + String nameOfType = nonNull.isEmpty() ? "null" : nonNull.iterator().next(); + switch (nameOfType) { + case "int": + type = ConnectorType.of("INT"); + break; + case "long": + type = ConnectorType.of("BIGINT"); + break; + case "double": + type = ConnectorType.of("DOUBLE"); + break; + case "bool": + type = ConnectorType.of("BOOLEAN"); + break; + case "string": + type = ConnectorType.of("STRING"); + break; + case "objectId": + type = ConnectorType.of("VARBINARY", 12, -1); + break; + case "date": + type = ConnectorType.of("DATETIMEV2", 3, 0); + break; + case "object": + children = objectFields(definition, depth, count); + type = children.isEmpty() ? ConnectorType.of("JSONB") : structType(children); + break; + case "array": + // MongoDB also permits tuple items (an array of schemas). Report that grammar + // as unsupported so auto takes the same whole-document fallback as other unions. + Object item = definition.get("items"); + if (!(item instanceof Document)) { + throw new IllegalArgumentException("MongoDB array requires homogeneous items"); + } + Binding element = validatorBinding("element", (Document) item, depth + 1, count); + children = List.of(element); + type = ConnectorType.arrayOf(element.type); + break; + case "timestamp": + children = List.of(longBinding("seconds"), longBinding("increment")); + type = structType(children); + break; + default: + // Decimal128 has no collection-wide scale; JSON preserves its exponent and special values. + type = ConnectorType.of("JSONB"); + } + } + return new Binding(name, List.of(name), type, tags, children, -1, ""); + } + + private static Binding explicitBinding(Document definition, int depth, int[] count) { + bounded(depth, count); + String name = definition.getString("name"); + List path = definition.getList("path", String.class); + if (name == null || name.isEmpty() || path == null || path.isEmpty() + || path.size() > MAX_DEPTH || path.stream().anyMatch(p -> p == null || p.isEmpty())) { + throw new IllegalArgumentException("MongoDB explicit column requires name and path segments"); + } + ConnectorType type = parseDorisType(definition.getString("doris_type")); + List sources = definition.getList("bson_types", String.class); + List sourceTags = sources == null ? List.of() : tags(sources); + int subtype = definition.getInteger("subtype", -1); + if (subtype < -1 || subtype > 255) { + throw new IllegalArgumentException("Invalid BSON binary subtype"); + } + List children = defaultChildren(type, depth, count); + if (definition.containsKey("children")) { + children = new ArrayList<>(); + for (Document child : definition.getList("children", Document.class)) { + children.add(explicitBinding(child, depth + 1, count)); + } + if (children.size() != type.getChildren().size()) { + throw new IllegalArgumentException("Explicit child mapping must match the Doris type"); + } + for (int i = 0; i < children.size(); i++) { + if (!children.get(i).type.equals(type.getChildren().get(i)) + || ("STRUCT".equals(type.getTypeName()) + && !children.get(i).name.equals(type.getFieldNames().get(i)))) { + throw new IllegalArgumentException("Explicit child mapping type mismatch"); + } + } + } + return new Binding(name, path, type, sourceTags, children, subtype, ""); + } + + private static List defaultChildren(ConnectorType type, int depth, int[] count) { + List children = new ArrayList<>(); + for (int i = 0; i < type.getChildren().size(); i++) { + bounded(depth + 1, count); + String name = type.getFieldNames().isEmpty() ? "element" : type.getFieldNames().get(i); + ConnectorType child = type.getChildren().get(i); + children.add(new Binding(name, List.of(name), child, List.of(), + defaultChildren(child, depth + 1, count), -1, "")); + } + return children; + } + + private static Binding longBinding(String name) { + return new Binding(name, List.of(name), ConnectorType.of("BIGINT"), List.of(18), List.of(), -1, ""); + } + + private static ConnectorType structType(List children) { + return ConnectorType.structOf(children.stream().map(b -> b.name).collect(Collectors.toList()), + children.stream().map(b -> b.type).collect(Collectors.toList())); + } + + private static List tags(List names) { + List result = new ArrayList<>(); + for (String name : names) { + Integer tag = BSON_TYPES.get(name); + if (tag == null) { + throw new IllegalArgumentException("Unsupported BSON schema type"); + } + result.add(tag); + } + return result; + } + + private static void bounded(int depth, int[] count) { + if (depth > MAX_DEPTH || ++count[0] > MAX_FIELDS) { + throw new IllegalArgumentException("MongoDB schema exceeds depth or field limits"); + } + } + + private static void unique(List bindings) { + Set names = new HashSet<>(); + for (Binding b : bindings) { + if (!names.add(b.name)) { + throw new IllegalArgumentException("MongoDB columns collide after case folding"); + } + } + } + + static ConnectorType parseDorisType(String text) { + if (text == null || text.length() > 8192) { + throw new IllegalArgumentException("Missing or oversized Doris type"); + } + TypeParser parser = new TypeParser(text); + ConnectorType type = parser.parse(0); + parser.space(); + if (parser.position != text.length()) { + throw new IllegalArgumentException("Trailing input in Doris type"); + } + return type; + } + + private static final class TypeParser { + private final String text; + private int position; + + TypeParser(String text) { + this.text = text; + } + + void space() { + while (position < text.length() && Character.isWhitespace(text.charAt(position))) { + position++; + } + } + + String word() { + space(); + int start = position; + while (position < text.length() && (Character.isLetterOrDigit(text.charAt(position)) + || text.charAt(position) == '_')) { + position++; + } + if (start == position) { + throw new IllegalArgumentException("Invalid Doris type syntax"); + } + return text.substring(start, position); + } + + boolean take(char token) { + space(); + if (position < text.length() && text.charAt(position) == token) { + position++; + return true; + } + return false; + } + + void expect(char token) { + if (!take(token)) { + throw new IllegalArgumentException("Invalid Doris type delimiter"); + } + } + + ConnectorType parse(int depth) { + if (depth > MAX_DEPTH) { + throw new IllegalArgumentException("Doris type exceeds nesting limit"); + } + String name = word().toUpperCase(Locale.ROOT); + if ("ARRAY".equals(name)) { + expect('<'); + ConnectorType child = parse(depth + 1); + expect('>'); + return ConnectorType.arrayOf(child); + } + if ("STRUCT".equals(name)) { + expect('<'); + List names = new ArrayList<>(); + List types = new ArrayList<>(); + do { + String field = word().toLowerCase(Locale.ROOT); + if (names.contains(field)) { + throw new IllegalArgumentException("Duplicate STRUCT field"); + } + names.add(field); + expect(':'); + types.add(parse(depth + 1)); + } while (take(',')); + expect('>'); + return ConnectorType.structOf(names, types); + } + int precision = -1; + int scale = -1; + if (take('(')) { + precision = Integer.parseInt(word()); + if (take(',')) { + scale = Integer.parseInt(word()); + } + expect(')'); + } + if ("DECIMAL".equals(name) || "DECIMALV3".equals(name)) { + if (precision < 1 || precision > 38 || scale < 0 || scale > precision) { + throw new IllegalArgumentException("MongoDB decimal requires DECIMAL(p,s), p <= 38"); + } + return ConnectorType.of("DECIMALV3", precision, scale); + } + if ("DATETIME".equals(name) || "DATETIMEV2".equals(name)) { + if (precision < 0 || precision > 6 || scale != -1) { + throw new IllegalArgumentException("MongoDB date requires DATETIME(scale), scale <= 6"); + } + return ConnectorType.of("DATETIMEV2", precision, 0); + } + if ("VARBINARY".equals(name)) { + if (precision == 0 || precision > 16777216 || scale != -1) { + throw new IllegalArgumentException("Invalid VARBINARY length"); + } + return ConnectorType.of(name, precision, -1); + } + if (precision != -1 || scale != -1 || !Set.of("BOOLEAN", "TINYINT", "SMALLINT", "INT", "BIGINT", + "LARGEINT", "FLOAT", "DOUBLE", "STRING", "JSON", "JSONB").contains(name)) { + throw new IllegalArgumentException("Unsupported explicit MongoDB Doris type"); + } + return ConnectorType.of("JSON".equals(name) ? "JSONB" : name); + } + } + + List getBindings() { + return bindings; + } + + List getColumns() { + return Collections.unmodifiableList(bindings.stream().map(Binding::column).collect(Collectors.toList())); + } + + String getSource() { + return source; + } + + String getVersion() { + return version; + } + + String getFallbackReason() { + return fallbackReason; + } + + String toBindingsJson() { + return new Document("version", 1).append("columns", + bindings.stream().map(Binding::toDocument).collect(Collectors.toList())).toJson(); + } + + static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoTableHandle.java b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoTableHandle.java new file mode 100644 index 00000000000000..1a113709785b56 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoTableHandle.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; + +import org.bson.Document; +import org.bson.json.JsonMode; +import org.bson.json.JsonWriterSettings; +import org.bson.types.Binary; + +import java.util.HexFormat; + +/** Namespace and frozen metadata values only; no MongoDB clients, sessions, or cursors. */ +final class MongoTableHandle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + private final String database; + private final String name; + private final String objectKind; + private final String metadataJson; + private final String uuid; + private final String schemaJson; + private transient volatile MongoSchema schema; + + MongoTableHandle(String database, Document metadata, MongoCatalogProperties properties) { + this.database = database; + metadataJson = metadata.toJson(JsonWriterSettings.builder().outputMode(JsonMode.EXTENDED).build()); + Document pinned = Document.parse(metadataJson); + name = pinned.getString("name"); + objectKind = pinned.getString("type"); + Object identity = pinned.get("info", new Document()).get("uuid"); + uuid = identity instanceof Binary ? HexFormat.of().formatHex(((Binary) identity).getData()) : ""; + schema = MongoSchema.resolve(database + "." + name, pinned, properties); + schemaJson = schema.toPinnedJson(); + } + + String getDatabase() { + return database; + } + + String getName() { + return name; + } + + String getObjectKind() { + return objectKind; + } + + String getUuid() { + return uuid; + } + + Document getOptions() { + return Document.parse(metadataJson).get("options", new Document()); + } + + MongoSchema getSchema() { + if (schema == null) { + schema = MongoSchema.fromPinnedJson(schemaJson); + } + return schema; + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider b/fe/fe-connector/fe-connector-mongodb/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider new file mode 100644 index 00000000000000..e66d579e0eb055 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ServiceLoader ignores hash comments, allowing the license in the provider descriptor. +org.apache.doris.connector.mongodb.MongoConnectorProvider diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoCatalogPropertiesTest.java new file mode 100644 index 00000000000000..b775ab9d6dd5fe --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoCatalogPropertiesTest.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import com.mongodb.ConnectionString; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class MongoCatalogPropertiesTest { + @Test + void appliesBoundedDefaultsAndRedactsDiagnostics() { + MongoCatalogProperties properties = MongoCatalogProperties.of(Map.of( + "mongodb.uri", "mongodb://localhost:27017", + "mongodb.username", "fixture_reader", "mongodb.password", "fixture_secret")); + Assertions.assertEquals("auto", properties.getSchemaMode()); + Assertions.assertEquals("primary", properties.getReadPreference()); + Assertions.assertEquals("majority", properties.getReadConcern()); + Assertions.assertFalse(properties.toString().contains("fixture_secret")); + Assertions.assertFalse(properties.toString().contains("mongodb://")); + } + + @Test + void preservesUriReadPreferenceTagsAndMaxStaleness() { + String uri = "mongodb://localhost:27017/?readPreference=secondary" + + "&readPreferenceTags=region:analytics&readPreferenceTags=&maxStalenessSeconds=90"; + MongoCatalogProperties properties = MongoCatalogProperties.of(Map.of( + "mongodb.uri", uri, "mongodb.read_preference", "nearest")); + Assertions.assertEquals("secondary", properties.getReadPreference()); + Assertions.assertEquals(new ConnectionString(uri).getReadPreference(), + properties.clientSettings().getReadPreference()); + } + + @Test + void rejectsConflictingCredentialSources() { + Assertions.assertThrows(IllegalArgumentException.class, () -> MongoCatalogProperties.of(Map.of( + "mongodb.uri", "mongodb://fixture_user:fixture_secret@localhost:27017", + "mongodb.username", "another_fixture"))); + } + + @Test + void malformedUriDoesNotLeakItsInput() { + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> MongoCatalogProperties.of(Map.of("mongodb.uri", "bad_fixture_secret"))); + Assertions.assertFalse(error.toString().contains("bad_fixture_secret")); + } + + @Test + void rejectsUnimplementedModesAndInvalidResourceBounds() { + for (Map.Entry invalid : Map.of( + "mongodb.schema.mode", "inferred", "mongodb.type_mismatch", "skip", + "mongodb.read_concern", "snapshot", "mongodb.pool_max_clients", "0", + "mongodb.socket_timeout_ms", "-1", "mongodb.datetime_timezone", "Asia/Shanghai").entrySet()) { + Map properties = new HashMap<>(); + properties.put("mongodb.uri", "mongodb://localhost:27017"); + properties.put(invalid.getKey(), invalid.getValue()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MongoCatalogProperties.of(properties), invalid.getKey()); + } + } + + @Test + void boundsExplicitSchemaAndRejectsUnknownMongoProperties() { + Assertions.assertThrows(IllegalArgumentException.class, () -> MongoCatalogProperties.of(Map.of( + "mongodb.uri", "mongodb://localhost:27017", "mongodb.unknown", "value"))); + Assertions.assertThrows(IllegalArgumentException.class, () -> MongoCatalogProperties.of(Map.of( + "mongodb.uri", "mongodb://localhost:27017", "mongodb.schema", "x".repeat(1_048_577)))); + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClientTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClientTest.java new file mode 100644 index 00000000000000..20881f6e9c2678 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClientTest.java @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.DorisConnectorException; + +import com.mongodb.MongoCommandException; +import com.mongodb.ServerAddress; +import com.mongodb.client.ListCollectionsIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.internal.operation.BatchCursor; +import com.mongodb.internal.operation.ListCollectionsOperation; +import org.bson.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class MongoDriverCatalogClientTest { + private static final class RecordingDriver { + final List commands = new ArrayList<>(); + int clientCloses; + int cursorCloses; + final List> operations = new ArrayList<>(); + boolean failGetMore; + boolean overflow; + long remainingCursor = 123; + + @SuppressWarnings("unchecked") + T proxy(Class type, java.lang.reflect.InvocationHandler handler) { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, handler); + } + + MongoClient client() { + MongoDatabase database = proxy(MongoDatabase.class, (p, method, args) -> { + if (method.getName().equals("listCollections")) { + return proxy(ListCollectionsIterable.class, (iterable, operation, parameters) -> { + if (operation.getName().equals("iterator")) { + throw new MongoCommandException(new Document("ok", 0).append("code", 13) + .append("errmsg", "fixture-secret").toBsonDocument(), new ServerAddress()); + } + return iterable; + }); + } + if (method.getName().startsWith("withRead")) { + return p; + } + if (method.getName().equals("runCommand")) { + commands.add((Document) args[args.length - 1]); + throw new MongoCommandException(new Document("ok", 0).append("code", 43) + .append("errmsg", "cursor belongs to another router").toBsonDocument(), new ServerAddress()); + } + throw new UnsupportedOperationException(method.getName()); + }); + return proxy(MongoClient.class, (p, method, args) -> { + switch (method.getName()) { + case "getDatabase": + return database; + case "close": + clientCloses++; + return null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + @SuppressWarnings("unchecked") + BatchCursor execute(ListCollectionsOperation operation) { + operations.add(operation); + int[] batch = {0}; + return proxy(BatchCursor.class, (cursor, method, args) -> { + switch (method.getName()) { + case "hasNext": + if (failGetMore && batch[0] == 1 && remainingCursor != 0) { + throw new MongoCommandException(new Document("ok", 0).append("code", 13) + .append("errmsg", "fixture-secret").toBsonDocument(), new ServerAddress()); + } + return batch[0] < (remainingCursor == 0 ? 1 : 2); + case "next": + if (overflow) { + return Collections.nCopies(100001, new Document("name", "first")); + } + return List.of(new Document("name", batch[0]++ == 0 ? "first" : "second") + .append("type", batch[0] == 1 ? "collection" : "view")); + case "close": + cursorCloses++; + return null; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + } + + MongoDriverCatalogClient catalog() { + return new MongoDriverCatalogClient(client(), MongoCatalogProperties.of( + Map.of("mongodb.uri", "mongodb://localhost:27017", "mongodb.databases", "sample")), + this::execute); + } + } + + @Test + void authorizedListingDrainsOneServerBoundCursorAndClosesIt() { + RecordingDriver driver = new RecordingDriver(); + try (MongoDriverCatalogClient catalog = driver.catalog()) { + Assertions.assertEquals(List.of("sample"), catalog.databases()); + List objects = catalog.collections("sample"); + Assertions.assertEquals(2, objects.size()); + Assertions.assertEquals("view", objects.get(1).getString("type")); + Assertions.assertEquals(1, driver.operations.size()); + Assertions.assertTrue(driver.operations.get(0).isAuthorizedCollections()); + Assertions.assertTrue(driver.operations.get(0).isNameOnly()); + Assertions.assertEquals(128, driver.operations.get(0).getBatchSize()); + Assertions.assertEquals(1, driver.cursorCloses); + Assertions.assertTrue(driver.commands.isEmpty()); + } + Assertions.assertEquals(1, driver.clientCloses); + } + + @Test + void failedGetMoreClosesOwningCursorAndRedactsSourceError() { + RecordingDriver driver = new RecordingDriver(); + driver.failGetMore = true; + try (MongoDriverCatalogClient catalog = driver.catalog()) { + DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class, + () -> catalog.collections("sample")); + Assertions.assertFalse(error.toString().contains("fixture-secret")); + Assertions.assertNull(error.getCause()); + Assertions.assertEquals(1, driver.cursorCloses); + Assertions.assertTrue(driver.commands.isEmpty()); + } + } + + @Test + void exhaustedCursorClosesWithoutReselectingAServer() { + RecordingDriver driver = new RecordingDriver(); + driver.remainingCursor = 0; + try (MongoDriverCatalogClient catalog = driver.catalog()) { + Assertions.assertEquals(1, catalog.collections("sample").size()); + Assertions.assertTrue(driver.commands.isEmpty()); + Assertions.assertEquals(1, driver.cursorCloses); + } + } + + @Test + void metadataLimitFailureClosesTheServerBoundCursor() { + RecordingDriver driver = new RecordingDriver(); + driver.overflow = true; + try (MongoDriverCatalogClient catalog = driver.catalog()) { + Assertions.assertThrows(DorisConnectorException.class, () -> catalog.collections("sample")); + Assertions.assertEquals(1, driver.cursorCloses); + } + } + + @Test + void autoFallsBackWithoutValidatorPrivilegesButForcedValidatorFails() { + RecordingDriver driver = new RecordingDriver(); + driver.remainingCursor = 0; + try (MongoDriverCatalogClient catalog = driver.catalog()) { + Document result = catalog.collection("sample", "first").orElseThrow(); + Assertions.assertFalse(result.getBoolean("metadata_available")); + Assertions.assertEquals("collection", result.getString("type")); + Assertions.assertFalse(result.containsKey("info")); + Assertions.assertEquals("first", driver.operations.get(0).getFilter().getString("name").getValue()); + Assertions.assertEquals(1, driver.cursorCloses); + } + RecordingDriver strict = new RecordingDriver(); + try (MongoDriverCatalogClient catalog = new MongoDriverCatalogClient(strict.client(), + MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017", + "mongodb.schema.mode", "validator")), strict::execute)) { + Assertions.assertThrows(DorisConnectorException.class, () -> catalog.collection("sample", "first")); + Assertions.assertTrue(strict.operations.isEmpty()); + } + } + +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoMetadataTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoMetadataTest.java new file mode 100644 index 00000000000000..1de5c718439034 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoMetadataTest.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorTableSchema; + +import org.bson.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +class MongoMetadataTest { + private static final class RecordingClient implements MongoCatalogClient { + private final List objects = new ArrayList<>(List.of( + new Document("name", "records").append("type", "collection"), + new Document("name", "projected").append("type", "view") + .append("options", new Document("viewOn", "records").append("pipeline", List.of())), + new Document("name", "measurements").append("type", "timeseries") + .append("options", new Document("timeseries", new Document("timeField", "time") + .append("metaField", "meta"))), + new Document("name", "system.buckets.measurements").append("type", "collection"))); + private int metadataRequests; + + @Override + public List databases() { + return List.of("sample"); + } + + @Override + public List collections(String database) { + metadataRequests++; + return objects; + } + + @Override + public void close() { + } + } + + private static MongoConnectorMetadata metadata(RecordingClient client) { + return new MongoConnectorMetadata(client, + MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017"))); + } + + @Test + void exposesLogicalObjectsAndHidesPhysicalBuckets() { + RecordingClient client = new RecordingClient(); + MongoConnectorMetadata metadata = metadata(client); + Assertions.assertEquals(List.of("sample"), metadata.listDatabaseNames(null)); + Assertions.assertEquals(List.of("records", "projected", "measurements"), + metadata.listTableNames(null, "sample")); + Assertions.assertFalse(metadata.getTableHandle(null, "sample", "system.buckets.measurements").isPresent()); + for (String name : List.of("records", "projected", "measurements")) { + MongoTableHandle handle = (MongoTableHandle) metadata.getTableHandle(null, "sample", name).orElseThrow(); + Assertions.assertTrue(handle.getUuid().isEmpty()); + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + Assertions.assertFalse(schema.getColumns().isEmpty()); + Assertions.assertFalse(schema.getProperties().containsKey("mongodb.uri")); + } + } + + @Test + void retainsViewAndTimeSeriesOptionsWithoutReadingBaseCollections() { + RecordingClient client = new RecordingClient(); + MongoConnectorMetadata metadata = metadata(client); + MongoTableHandle view = (MongoTableHandle) metadata.getTableHandle(null, "sample", "projected").orElseThrow(); + MongoTableHandle timeSeries = (MongoTableHandle) metadata.getTableHandle(null, "sample", "measurements") + .orElseThrow(); + Assertions.assertEquals("view", view.getObjectKind()); + Assertions.assertEquals("records", view.getOptions().getString("viewOn")); + Assertions.assertEquals("timeseries", timeSeries.getObjectKind()); + Assertions.assertEquals("time", timeSeries.getOptions().get("timeseries", Document.class) + .getString("timeField")); + Assertions.assertEquals(2, client.metadataRequests); + } + + @Test + void oldHandlesRemainPinnedWhenMetadataChanges() { + RecordingClient client = new RecordingClient(); + MongoConnectorMetadata metadata = metadata(client); + MongoTableHandle first = (MongoTableHandle) metadata.getTableHandle(null, "sample", "records").orElseThrow(); + client.objects.get(0).append("options", Document.parse("{\"validator\":{\"$jsonSchema\":{" + + "\"properties\":{\"value\":{\"bsonType\":\"int\"}}}}}")); + MongoTableHandle next = (MongoTableHandle) metadata.getTableHandle(null, "sample", "records").orElseThrow(); + Assertions.assertNotEquals(first.getSchema().getVersion(), next.getSchema().getVersion()); + Assertions.assertEquals("DOCUMENT", first.getSchema().getSource()); + Assertions.assertEquals("MONGODB_VALIDATOR", next.getSchema().getSource()); + } + + @Test + void serializedHandleContainsStableValuesAndRestoresItsSchema() throws Exception { + MongoTableHandle handle = (MongoTableHandle) metadata(new RecordingClient()) + .getTableHandle(null, "sample", "records").orElseThrow(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(handle); + } + try (ObjectInputStream input = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + MongoTableHandle restored = (MongoTableHandle) input.readObject(); + Assertions.assertEquals(handle.getSchema().getVersion(), restored.getSchema().getVersion()); + Assertions.assertEquals(handle.getSchema().getColumns(), restored.getSchema().getColumns()); + } + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoProviderTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoProviderTest.java new file mode 100644 index 00000000000000..5d2a8bc023c115 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoProviderTest.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorCapability; +import org.apache.doris.connector.spi.ConnectorProvider; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; + +class MongoProviderTest { + @Test + void serviceRegistrationAndSecuritySurfaceAreComplete() { + ConnectorProvider provider = ServiceLoader.load(ConnectorProvider.class).stream() + .map(ServiceLoader.Provider::get).filter(p -> "mongodb".equals(p.getType())).findFirst().orElseThrow(); + Assertions.assertEquals(List.of("mongodb.uri"), provider.securitySurface().ssrfEndpointKeys()); + Assertions.assertThrows(IllegalArgumentException.class, () -> provider.validateProperties(Map.of())); + Assertions.assertTrue(MongoConnectorMetadata.TABLE_NAMESPACE.startsWith(provider.getType() + ".")); + } + + @Test + void constructionDoesNotOpenConnectionsAndSourceViewsRemainNativeTables() throws Exception { + try (Connector connector = new MongoConnectorProvider().create( + Map.of("mongodb.uri", "mongodb://localhost:1"), null)) { + Assertions.assertNotNull(connector.getScanPlanProvider()); + Assertions.assertFalse(connector.getCapabilities().contains(ConnectorCapability.SUPPORTS_VIEW)); + } + } +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoScanPlanTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoScanPlanTest.java new file mode 100644 index 00000000000000..786544ade6a9b2 --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoScanPlanTest.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.scan.ConnectorScanRange; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.scan.ScanNodePropertiesResult; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.bson.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +class MongoScanPlanTest { + private static final class Session implements ConnectorSession { + @Override + public String getQueryId() { + return "fixture-query"; + } + + @Override + public String getUser() { + return "fixture-user"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en"; + } + + @Override + public long getCatalogId() { + return 42; + } + + @Override + public String getCatalogName() { + return "fixture-catalog"; + } + + @Override + public T getProperty(String name, Class type) { + throw new UnsupportedOperationException("Query policy must use session properties directly"); + } + + @Override + public Map getCatalogProperties() { + return Map.of(); + } + + @Override + public Map getSessionProperties() { + return Map.of("batch_size", "128", "query_timeout", "30"); + } + } + + private static MongoCatalogProperties properties() { + return MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017", + "mongodb.username", "fixture-user", "mongodb.password", "fixture-secret")); + } + + @Test + void emitsOneStableNativeRangeForEachLogicalObjectKind() { + MongoCatalogProperties properties = properties(); + MongoScanPlanProvider provider = new MongoScanPlanProvider(properties); + for (String kind : List.of("collection", "view", "timeseries")) { + MongoTableHandle handle = new MongoTableHandle("sample", + new Document("name", "records").append("type", kind), properties); + ConnectorScanRequest request = ConnectorScanRequest.builder(handle, List.of()).build(); + List ranges = provider.planScan(new Session(), request); + Assertions.assertEquals(1, ranges.size()); + ConnectorScanRange range = ranges.get(0); + Assertions.assertEquals("mongodb", range.getTableFormatType()); + Assertions.assertEquals("mongodb", range.getFileFormat()); + Assertions.assertTrue(range.isNativeReadRange()); + Assertions.assertTrue(range.getProperties().isEmpty()); + TTableFormatFileDesc descriptor = new TTableFormatFileDesc(); + range.populateRangeParams(descriptor, new TFileRangeDesc()); + Assertions.assertFalse(descriptor.isSetJdbcParams()); + Assertions.assertEquals(1, descriptor.getMongodbParams().getVersion()); + Assertions.assertEquals(kind, descriptor.getMongodbParams().getObjectKind()); + Assertions.assertEquals(handle.getSchema().getVersion(), descriptor.getMongodbParams().getSchemaVersion()); + Assertions.assertEquals(128, descriptor.getMongodbParams().getBatchSize()); + Assertions.assertEquals(30000, descriptor.getMongodbParams().getTimeoutMs()); + } + } + + @Test + void explainContainsNoCredentialsAndDoesNotClaimPredicatePushdown() { + MongoCatalogProperties properties = properties(); + MongoTableHandle handle = new MongoTableHandle("sample", + new Document("name", "records").append("type", "collection"), properties); + MongoScanPlanProvider provider = new MongoScanPlanProvider(properties); + ScanNodePropertiesResult result = provider.getScanNodePropertiesResult(new Session(), handle, + List.of(), Optional.empty()); + Assertions.assertFalse(result.hasConjunctTracking()); + Assertions.assertFalse(result.getProperties().toString().contains("fixture-secret")); + Assertions.assertFalse(result.getProperties().toString().contains("mongodb://")); + } + + @Test + void columnHandlesRetainBsonIdentityAndRejectAnotherSchemaVersion() { + MongoCatalogProperties properties = properties(); + MongoTableHandle handle = new MongoTableHandle("sample", + new Document("name", "records").append("type", "collection"), properties); + MongoSchema.Binding binding = handle.getSchema().getBindings().get(0); + MongoColumnHandle current = new MongoColumnHandle(binding, handle.getSchema().getVersion()); + MongoColumnHandle stale = new MongoColumnHandle(binding, "another-schema"); + Assertions.assertNotEquals(current, stale); + Assertions.assertEquals(current, new MongoColumnHandle(binding, handle.getSchema().getVersion())); + Assertions.assertTrue(current.getBindingJson().contains("bson_types")); + Assertions.assertThrows(IllegalArgumentException.class, () -> new MongoScanPlanProvider(properties) + .planScan(new Session(), ConnectorScanRequest.builder(handle, List.of(stale)).build())); + } + +} diff --git a/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoSchemaTest.java b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoSchemaTest.java new file mode 100644 index 00000000000000..021b24a1f1012e --- /dev/null +++ b/fe/fe-connector/fe-connector-mongodb/src/test/java/org/apache/doris/connector/mongodb/MongoSchemaTest.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.mongodb; + +import org.apache.doris.connector.spi.ConnectorType; + +import org.bson.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +class MongoSchemaTest { + private static Document collection(String fields) { + return Document.parse(("{name:'records',type:'collection',options:{validator:{$jsonSchema:" + + "{bsonType:'object',properties:" + fields + "}}}}").replace('\'', '"')); + } + + private static MongoSchema resolve(String mode, Document metadata) { + return MongoSchema.resolve("sample.records", metadata, + MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017", + "mongodb.schema.mode", mode))); + } + + @Test + void mapsValidatorWithoutPromisingHistoricalNonNullValues() { + MongoSchema schema = resolve("validator", collection("{_id:{bsonType:'objectId'}," + + "sequence:{bsonType:['int','long','null']},label:{bsonType:'string'}}")); + Assertions.assertEquals("MONGODB_VALIDATOR", schema.getSource()); + Assertions.assertEquals(ConnectorType.of("VARBINARY", 12, -1), schema.getColumns().get(0).getType()); + Assertions.assertEquals(ConnectorType.of("BIGINT"), schema.getColumns().get(1).getType()); + Assertions.assertTrue(schema.getColumns().stream().allMatch(c -> c.isNullable())); + } + + @Test + void mapsNestedObjectsAndArraysWithoutSampling() { + MongoSchema schema = resolve("validator", collection("{profile:{bsonType:'object',properties:" + + "{City:{bsonType:'string'}}},scores:{bsonType:'array',items:{bsonType:'long'}}}")); + Assertions.assertEquals(List.of("city"), schema.getColumns().get(0).getType().getFieldNames()); + Assertions.assertEquals(ConnectorType.arrayOf(ConnectorType.of("BIGINT")), + schema.getColumns().get(1).getType()); + Assertions.assertTrue(schema.toBindingsJson().contains("City")); + } + + @Test + void unsupportedValidatorFallsBackAsAWhole() { + Document metadata = collection("{known:{bsonType:'int'},unknown:{anyOf:[{bsonType:'int'}]}}"); + MongoSchema schema = resolve("auto", metadata); + Assertions.assertEquals("DOCUMENT", schema.getSource()); + Assertions.assertEquals(List.of("_mongo_ejson", "_mongo_bson"), + schema.getColumns().stream().map(c -> c.getName()).collect(java.util.stream.Collectors.toList())); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve("validator", metadata)); + } + + @Test + void tupleArrayValidatorFallsBackAsAWhole() { + Document metadata = collection("{known:{bsonType:'int'},values:{bsonType:'array'," + + "items:[{bsonType:'int'},{bsonType:'string'}]}}"); + Assertions.assertEquals("DOCUMENT", resolve("auto", metadata).getSource()); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve("validator", metadata)); + } + + @Test + void schemaBearingKeywordsNeverPublishPartialFields() { + for (String keyword : List.of( + "{patternProperties:{'^extra$':{bsonType:'string'}}}", + "{dependencies:{known:{properties:{extra:{bsonType:'string'}}}}}", + "{additionalProperties:{bsonType:'string'}}", + "{not:{properties:{extra:{bsonType:'string'}}}}")) { + Document extension = Document.parse(keyword.replace('\'', '"')); + Document root = collection("{known:{bsonType:'int'}}"); + root.get("options", Document.class).get("validator", Document.class) + .get("$jsonSchema", Document.class).putAll(extension); + Assertions.assertEquals("DOCUMENT", resolve("auto", root).getSource(), keyword); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve("validator", root), keyword); + Document nested = collection("{profile:{bsonType:'object',properties:{known:{bsonType:'int'}}}}"); + nested.get("options", Document.class).get("validator", Document.class) + .get("$jsonSchema", Document.class).get("properties", Document.class) + .get("profile", Document.class).putAll(extension); + Assertions.assertEquals("DOCUMENT", resolve("auto", nested).getSource(), keyword); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve("validator", nested), keyword); + } + } + + @Test + void ordinaryObjectConstraintsKeepTypedSchema() { + Document root = collection("{known:{bsonType:'int'}}"); + root.get("options", Document.class).get("validator", Document.class) + .get("$jsonSchema", Document.class).append("additionalProperties", false) + .append("patternProperties", new Document()).append("dependencies", new Document("known", List.of("known"))); + Assertions.assertEquals("MONGODB_VALIDATOR", resolve("auto", root).getSource()); + } + + @Test + void viewsAndTimeSeriesUseTheirOutputSchema() { + Document metadata = collection("{underlying:{bsonType:'int'}}"); + for (String kind : List.of("view", "timeseries")) { + metadata.put("type", kind); + Assertions.assertEquals("DOCUMENT", resolve("auto", metadata).getSource()); + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve("validator", metadata)); + } + } + + @Test + void explicitPathsPreserveLiteralDotsAndDeclaredLogicalTypes() { + String manifest = "{\"sample.records\":{\"columns\":[" + + "{\"name\":\"identifier\",\"path\":[\"_id\"],\"bson_types\":[\"objectId\"]," + + "\"doris_type\":\"VARBINARY(12)\"}," + + "{\"name\":\"amount\",\"path\":[\"a.b\"],\"bson_types\":[\"decimal\"]," + + "\"doris_type\":\"DECIMAL(20,2)\"}]}}"; + MongoSchema schema = MongoSchema.resolve("sample.records", collection("{}"), + MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017", + "mongodb.schema", manifest))); + Assertions.assertEquals("DORIS_EXPLICIT", schema.getSource()); + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 20, 2), schema.getColumns().get(1).getType()); + Assertions.assertEquals(List.of("a.b"), schema.getBindings().get(1).getPath()); + } + + @Test + void rejectsAmbiguousColumnNamesAndUnsupportedExplicitTypes() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> resolve("validator", collection("{A:{bsonType:'int'},a:{bsonType:'int'}}"))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MongoSchema.parseDorisType("DECIMAL(39,2)")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MongoSchema.parseDorisType("DATETIME(7)")); + } + + @Test + void definitionChangesProduceNewPinnedVersions() { + Document metadata = collection("{n:{bsonType:'int'}}"); + MongoSchema first = resolve("auto", metadata); + metadata.get("options", Document.class).put("collation", new Document("locale", "en")); + Assertions.assertNotEquals(first.getVersion(), resolve("auto", metadata).getVersion()); + Assertions.assertEquals(first.getVersion(), resolve("auto", collection("{n:{bsonType:'int'}}")).getVersion()); + } + + @Test + void schemasDoNotInventKeysForSourceDocuments() { + MongoSchema schema = MongoSchema.resolve("sample.records", + new Document("name", "records").append("type", "view"), + MongoCatalogProperties.of(Map.of("mongodb.uri", "mongodb://localhost:27017"))); + Assertions.assertTrue(schema.getColumns().stream().noneMatch(c -> c.isKey())); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java index 4b6f1fce6afdcb..ddead9547ae830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DatasourcePrintableMap.java @@ -43,6 +43,9 @@ public class DatasourcePrintableMap extends BasicPrintableMap { SENSITIVE_KEY.add("kerberos_keytab_content"); SENSITIVE_KEY.add("bos_secret_accesskey"); SENSITIVE_KEY.add("jdbc.password"); + // MongoDB URIs can embed credentials even when no separate password is configured. + SENSITIVE_KEY.add("mongodb.uri"); + SENSITIVE_KEY.add("mongodb.password"); SENSITIVE_KEY.add("elasticsearch.password"); SENSITIVE_KEY.add("ai.api_key"); SENSITIVE_KEY.add("ai.embed.api_key"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 2ee7c5a4e8eacc..39f04ed38831cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -2806,6 +2806,8 @@ static TFileFormatType mapFileFormatType(String format) { return TFileFormatType.FORMAT_AVRO; case "es_http": return TFileFormatType.FORMAT_ES_HTTP; + case "mongodb": + return TFileFormatType.FORMAT_MONGODB; case "arrow": // A connector whose reader hands BE Arrow record batches rather than a file (adbc, and the // remote_doris scan node already outside this switch). Without this the format falls to diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java index 45d68bbf7543bb..304b7b18573122 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/DatasourcePrintableMapTest.java @@ -59,6 +59,15 @@ public void testSensitiveKeysContainAliyunDLFProperties() { Assertions.assertTrue(DatasourcePrintableMap.SENSITIVE_KEY.contains("oss.secret_key")); } + @Test + public void testMongoCredentialsAreMasked() { + String result = new DatasourcePrintableMap<>(Map.of( + "mongodb.uri", "mongodb://fixture-user:fixture-secret@localhost:27017", + "mongodb.password", "fixture-secret"), "=", true, true, true).toString(); + Assertions.assertFalse(result.contains("fixture-secret")); + Assertions.assertFalse(result.contains("mongodb://")); + } + @Test public void testBasicConstructor() { Map testMap = new HashMap<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java index 98e7457f5e1680..f0e5b8bada3b3e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeFileFormatTest.java @@ -57,6 +57,8 @@ public void mapsTheFileBackedFormatsToTheirNativeReaders() { PluginDrivenScanNode.mapFileFormatType("avro")); Assertions.assertEquals(TFileFormatType.FORMAT_ES_HTTP, PluginDrivenScanNode.mapFileFormatType("es_http")); + Assertions.assertEquals(TFileFormatType.FORMAT_MONGODB, + PluginDrivenScanNode.mapFileFormatType("mongodb")); } @Test diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 993c25c05e499e..9a9f99b68d55b5 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -132,7 +132,8 @@ enum TFileFormatType { FORMAT_TEXT = 17, FORMAT_NATIVE = 18, FORMAT_LANCE = 19, - FORMAT_ES_HTTP = 20 + FORMAT_ES_HTTP = 20, + FORMAT_MONGODB = 21 } // In previous versions, the data compression format and file format were stored together, as TFileFormatType, @@ -660,6 +661,64 @@ struct TLanceFileDesc { 6: optional bool use_scalar_index } +// Stable single-cursor read contract. Driver objects and active sessions remain local to FE/BE. +// The reader validates the required version-1 fields before opening the logical namespace. +struct TMongoScanDesc { + // Descriptor layout understood by the native reader; currently 1. + 1: optional i32 version + // Doris catalog identity used to isolate authenticated BE client pools. + 2: optional i64 catalog_id + // Fingerprint of pinned connection/authentication settings; pool reuse requires a matching value. + 3: optional string configuration_version + // Remote MongoDB database name, preserving the source spelling. + 4: optional string database + // Public logical collection/view/time-series name; never an internal system.buckets namespace. + 5: optional string collection + // Source object kind: "collection", "view", or "timeseries". + 6: optional string object_kind + // Available source collection UUID as lowercase hex. Empty means metadata did not provide one; + // a nonempty value must match before BE scans, preventing reads after DROP/CREATE name reuse. + 7: optional string collection_uuid + // Fingerprint of the statement-pinned schema/source definition, also carried by column handles. + 8: optional string schema_version + // Versioned JSON containing owned field names, exact path segments, BSON tags/subtypes and + // child bindings. BE uses it for decoding; it does not sample or infer a schema from rows. + 9: optional string bindings_json + // MongoDB connection URI (including optional TLS/SRV/compression settings). May contain secrets; + // keep it out of EXPLAIN, profiles, and error messages. + 10: optional string uri + // Optional separate authentication user; empty uses URI credentials or unauthenticated access. + 11: optional string username + // Secret for the separate username, carried only in the typed scan RPC descriptor. + 12: optional string password + // Authentication database for separate credentials, normally "admin"; not the data database. + 13: optional string auth_database + // Data-read routing: primary, primaryPreferred, secondary, secondaryPreferred, or nearest. + // URI tag sets and max staleness remain in uri and are preserved by the native driver. + // Metadata/UUID checks use primary independently of this data-read preference. + 14: optional string read_preference + // Data-read concern: local, majority, or available. Majority alone is not a shared snapshot. + 15: optional string read_concern + // Replace top-level conversion mismatches with SQL NULL after rollback. Transport, cancellation, + // allocation and document-limit errors still fail the scan. + 16: optional bool type_mismatch_as_null + // Positive document count requested per find/getMore driver batch (1..65536), from the session. + // Doris output blocks also obey a separate row/byte budget in MongoFileReader. + 17: optional i32 batch_size + // Native scan lifetime in milliseconds, derived from session query_timeout; bounds pool waits + // and driver operations and is also checked before/after advancing the cursor. + 18: optional i64 timeout_ms + // Connection establishment timeout in milliseconds; bounded by timeout_ms when creating a pool. + 19: optional i32 connect_timeout_ms + // Server-selection timeout in milliseconds; bounded by timeout_ms when creating a pool. + 20: optional i32 server_selection_timeout_ms + // Socket I/O timeout in milliseconds; the leased client is bounded by the remaining scan time. + 21: optional i32 socket_timeout_ms + // Maximum simultaneously leased clients in this pool, keyed by catalog/configuration and + // connection policy. This is not a query-wide or source-wide scanner-concurrency budget. + 22: optional i32 pool_max_clients +} + struct TTableFormatFileDesc { 1: optional string table_format_type 2: optional TIcebergFileDesc iceberg_params @@ -694,6 +753,7 @@ struct TTableFormatFileDesc { // straight to the Java scanner, so a typed struct would only add a transcription step (see // es_params, jdbc_params). 15: optional map fluss_params + 16: optional TMongoScanDesc mongodb_params } // Deprecated, hive text talbe is a special format, not a serde type diff --git a/regression-test/data/external_table_p0/mongodb/test_mongodb_native.out b/regression-test/data/external_table_p0/mongodb/test_mongodb_native.out new file mode 100644 index 00000000000000..97965bff3e9ffb --- /dev/null +++ b/regression-test/data/external_table_p0/mongodb/test_mongodb_native.out @@ -0,0 +1,350 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !discovery -- +boundary_errors +boundary_records +deep_limit_records +dependency_validator +empty_records +historical_records +large_records +measurements +mismatched +nested_pattern_validator +pattern_validator +record_view +special_records +tuple_validator +typed_records +unsupported_validator +wide_records + +-- !validator_schema -- +_id varbinary(12) Yes false \N +active boolean Yes false \N +amount json Yes false \N +blob json Yes false \N +clock struct Yes false \N +created datetime(3) Yes false \N +grp text Yes false \N +name text Yes false \N +profile struct Yes false \N +ratio double Yes false \N +score bigint Yes false \N +seq bigint Yes false \N +tags array Yes false \N + +-- !typed -- +1 alpha east 10 true 1.25 000000000000000000000001 1969-12-31T23:59:59.999 ["red", "blue"] A 1 000000000000000000000065 4294967295 4294967295 +2 beta west 20 false -0.5 000000000000000000000002 2024-02-01T12:34:56.789 [] B 2 \N \N \N +3 中文-é east \N true 0 000000000000000000000003 1970-01-01T00:00 [null, "中文"] \N \N \N \N \N +4 delta west \N false 2.5 000000000000000000000004 2038-01-19T03:14:07 \N \N \N \N \N \N + +-- !objectid_equal -- +1 + +-- !objectid_join -- +4 + +-- !objectid_groups -- +000000000000000000000001 1 +000000000000000000000002 1 +000000000000000000000003 1 +000000000000000000000004 1 + +-- !null -- +1 false false false +2 false false false +3 true true false +4 true true true + +-- !decimal_json -- +1 {"$numberDecimal":"123.4500"} {"$binary":{"base64":"AP+A","subType":"00"}} +2 {"$numberDecimal":"-2.50"} {"$binary":{"base64":"AP+A","subType":"80"}} +3 {"$numberDecimal":"0E-6176"} \N +4 {"$numberDecimal":"99999.99"} \N + +-- !utc_east -- +1 1969-12-31T23:59:59.999 +2 2024-02-01T12:34:56.789 +3 1970-01-01T00:00 +4 2038-01-19T03:14:07 + +-- !utc_west -- +1 1969-12-31T23:59:59.999 +2 2024-02-01T12:34:56.789 +3 1970-01-01T00:00 +4 2038-01-19T03:14:07 + +-- !where -- +1 alpha + +-- !or_in -- +1 +3 +4 + +-- !like -- +1 alpha +2 beta +4 delta + +-- !regexp -- +1 alpha +2 beta + +-- !unicode -- +3 + +-- !group -- +east 2 10 1 3 +west 2 20 2 4 + +-- !join -- +1 3 +2 4 + +-- !array -- +1 2 true +2 0 false +3 2 false +4 \N \N + +-- !window -- +1 1 +2 1 +3 2 +4 2 + +-- !order_limit -- +3 中文-é +2 beta + +-- !count -- +4 + +-- !empty -- +0 0 + +-- !zero_match -- + +-- !multi_batch -- +2081 0 2080 2164240 + +-- !batch_edges -- +0 +1023 +1024 +1025 +126 +127 +128 +2047 +2048 +2080 + +-- !batch_offset -- +1022 +1023 +1024 +1025 +1026 + +-- !repeated_limit_0 -- +0 +1 +2 + +-- !early_cursor_close_0 -- +1 + +-- !repeated_limit_1 -- +0 +1 +2 + +-- !early_cursor_close_1 -- +1 + +-- !repeated_limit_2 -- +0 +1 +2 + +-- !early_cursor_close_2 -- +1 + +-- !after_limits -- +2081 + +-- !fallback_schema_tuple_validator -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !fallback_values_tuple_validator -- +{"_id":{"$oid":"000000000000000000000001"},"known":{"$numberInt":"1"},"extra":"present","values":[{"$numberInt":"1"},"x"],"profile":{"known":{"$numberInt":"2"},"extra":"nested"}} + +-- !fallback_schema_pattern_validator -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !fallback_values_pattern_validator -- +{"_id":{"$oid":"000000000000000000000001"},"known":{"$numberInt":"1"},"extra":"present","values":[{"$numberInt":"1"},"x"],"profile":{"known":{"$numberInt":"2"},"extra":"nested"}} + +-- !fallback_schema_dependency_validator -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !fallback_values_dependency_validator -- +{"_id":{"$oid":"000000000000000000000001"},"known":{"$numberInt":"1"},"extra":"present","values":[{"$numberInt":"1"},"x"],"profile":{"known":{"$numberInt":"2"},"extra":"nested"}} + +-- !fallback_schema_nested_pattern_validator -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !fallback_values_nested_pattern_validator -- +{"_id":{"$oid":"000000000000000000000001"},"known":{"$numberInt":"1"},"extra":"present","values":[{"$numberInt":"1"},"x"],"profile":{"known":{"$numberInt":"2"},"extra":"nested"}} + +-- !fallback_schema -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !fallback -- +{"_id":{"$oid":"000000000000000000000001"},"value":{"$numberInt":"1"}} true + +-- !view_document -- +{"seq":{"$numberLong":"1"},"label":"view-alpha"} +{"seq":{"$numberLong":"3"},"label":"view-中文-é"} + +-- !timeseries_document -- +3 + +-- !document_override -- +_mongo_bson varbinary(2147483647) Yes false \N +_mongo_ejson json Yes false \N + +-- !document_bytes -- +4 130 265 + +-- !extended_bson -- +{"_id":{"$oid":"000000000000000000000001"},"integral":{"$numberLong":"9007199254740993"},"decimal":{"$numberDecimal":"1.2300"},"date":{"$date":{"$numberLong":"-1"}},"bytes":{"$binary":{"base64":"AP+A","subType":"80"}},"pattern":{"$regularExpression":{"pattern":"a.*b","options":"im"}},"clock":{"$timestamp":{"t":4294967295,"i":7}},"low":{"$minKey":1},"high":{"$maxKey":1},"code":{"$code":"return n","$scope":{"n":{"$numberInt":"1"}}},"refs":[{"$oid":"000000000000000000000002"},null],"literal.dot":"literal","nested":{"dot":"nested"},"dollar$field":"dollar","unicode":"中文"} + +-- !forced_validator -- +4 10 + +-- !wide_scalar -- +1 + +-- !wide_count -- +1 + +-- !view -- +1 view-alpha +3 view-中文-é + +-- !view_analysis -- +2 1 3 + +-- !timeseries -- +2024-01-01T00:00:00.001 a 1 +2024-01-01T00:00:00.001 a 2 +2024-01-02T00:00 b 3 + +-- !timeseries_where -- +a 2 3 + +-- !explicit_paths -- +000000000000000000000001 9007199254740993 1.2300 literal nested dollar + +-- !special_types -- +{"$regularExpression":{"pattern":"a.*b","options":"im"}} {"$minKey":1} {"$maxKey":1} {"$code":"return n","$scope":{"n":{"$numberInt":"1"}}} + +-- !refs -- +2 000000000000000000000002 true + +-- !type_nulls -- +1 true true true true true true true true true true true true true true true true true true true true true true true true true true true true +2 true true true true true true true true true true true true true true true true true true true true true true true true true true true true + +-- !integer_decimal_bounds -- +10 \N \N \N \N \N \N -0.01 0.0001 0.0001 +3 false -128 -32768 -2147483648 -9223372036854775808 -9223372036854775808 -9999999.99 -99999999999999.9999 -9999999999999999999999999999999999.0000 +4 true 127 32767 2147483647 9223372036854775807 9223372036854775807 9999999.99 99999999999999.9999 9999999999999999999999999999999999.0000 +5 false 0 0 0 0 0 0.00 0.0000 0.0000 + +-- !float_bounds -- +10 1.1754944e-38 2.2250738585072014e-308 +11 0 0 +3 -3.4028235e+38 -1.7976931348623157e+308 +4 3.4028235e+38 1.7976931348623157e+308 +5 1e-45 5e-324 +6 NaN NaN +7 Infinity Infinity +8 -Infinity -Infinity +9 -0 -0 + +-- !binary_string_bounds -- +3 000000000000000000000000 0 0 0 +4 FFFFFFFFFFFFFFFFFFFFFFFF 00FF807F 4 12 5 6100E4B8ADE69687F09F9880 +5 \N 00 1 4096 4096 6161616161616161 + +-- !datetime_timestamp_bounds -- +10 1970-01-01 00:00:00.000 0 \N \N \N +3 0000-01-01 00:00:00.000 -62167219200000 0000-01-01 00:00:00 0 0 +4 9999-12-31 23:59:59.999 253402300799999 9999-12-31 23:59:59 4294967295 4294967295 +5 1969-12-31 23:59:59.999 -1 1970-01-01 00:00:00 \N \N + +-- !complex_bounds -- +3 [] {"a":null, "b":""} [] [] +4 [null, -2147483648, 2147483647] {"a":2147483647, "b":"中文"} [null, {"a":null, "b":""}, {"a":2147483647, "b":null}] {} +5 [null] {"a":null, "b":null} [null, {"a":null, "b":null}] + +-- !special_bounds -- +10 \N \N \N \N {"$code":""} {"$code":"","$scope":{}} \N +11 \N \N \N \N \N \N \N +3 ^$ \N \N \N \N \N +4 中文.*é im {"$minKey":1} {"$maxKey":1} {"$code":"return 1"} {"$code":"return n","$scope":{"n":null}} \N +5 \N \N \N \N \N \N \N +6 \N \N \N \N \N \N {"$numberDecimal":"NaN"} +7 \N \N \N \N \N \N {"$numberDecimal":"9.999999999999999999999999999999999E+6144"} +8 \N \N \N \N \N \N {"$numberDecimal":"1E-6176"} +9 \N \N \N \N \N \N {"$numberDecimal":"-0.0000"} + +-- !epoch_outside_datetime_range -- +1 253402300800000 +2 -62167219200001 + +-- !refresh -- +nested nested + +-- !mismatch_nulls -- +1 10 [1, 2] {"a":1, "b":"ok"} 1.23 +2 \N \N \N \N +3 30 [4] {"a":3, "b":null} 2.00 + +-- !mismatch_filter -- +2 + +-- !boundary_error_nulls -- +1 true true true true true true true true true true true true true true true true true true true true +2 true true true true true true true true true true true true true true true true true true true true +3 true true true true true true true true true true true true true true true true true true true true + +-- !view_only_discovery -- +record_view + +-- !view_only_read -- +1 view-alpha +3 view-中文-é + +-- !metadata_pagination -- +page_130 +page_131 +page_132 +page_133 +page_134 +page_135 +page_136 +page_137 +page_138 +page_139 + diff --git a/regression-test/suites/external_table_p0/mongodb/test_mongodb_native.groovy b/regression-test/suites/external_table_p0/mongodb/test_mongodb_native.groovy new file mode 100644 index 00000000000000..dea5108c0c852a --- /dev/null +++ b/regression-test/suites/external_table_p0/mongodb/test_mongodb_native.groovy @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput + +suite("test_mongodb_native", "p0,external") { + if (!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableMongoDBTest"))) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + def create = { String name, Map extra -> + sql "DROP CATALOG IF EXISTS ${name}" + Map properties = ["type": "mongodb", "mongodb.uri": "mongodb://${host}:${port}", + "mongodb.username": user, "mongodb.password": password, "mongodb.databases": "doris_mongodb_native"] + properties.putAll(extra) + String values = properties.collect { key, value -> + "\"${key}\" = '${value.toString().replace("\\", "\\\\").replace("'", "\\'")}'" + }.join(",\n") + sql "CREATE CATALOG ${name} PROPERTIES (${values})" + } + create("mongodb_native_auto", [:]) + sql "SWITCH mongodb_native_auto" + sql "USE doris_mongodb_native" + + order_qt_discovery "SHOW TABLES" + order_qt_validator_schema "DESC typed_records" + order_qt_typed """SELECT seq, name, grp, score, active, ratio, from_binary(_id), created, + tags, profile.city, profile.rank, from_binary(profile.manager), clock.seconds, clock.increment + FROM typed_records""" + order_qt_objectid_equal "SELECT seq FROM typed_records WHERE FROM_BINARY(_id) = '000000000000000000000001'" + order_qt_objectid_join "SELECT count(*) FROM typed_records a JOIN typed_records b ON FROM_BINARY(a._id) = FROM_BINARY(b._id)" + order_qt_objectid_groups "SELECT from_binary(_id), count(*) FROM typed_records GROUP BY FROM_BINARY(_id)" + test { + sql "SELECT seq FROM typed_records WHERE _id = X'000000000000000000000001'" + exception "varbinary" + } + test { + sql "SELECT _id, count(*) FROM typed_records GROUP BY _id" + exception "don't support filter, group by or order by" + } + order_qt_null "SELECT seq, score IS NULL, profile IS NULL, tags IS NULL FROM typed_records" + order_qt_decimal_json "SELECT seq, cast(amount AS STRING), cast(blob AS STRING) FROM typed_records" + + def originalTimeZone = sql("SELECT @@time_zone")[0][0] + sql "SET time_zone = '+08:00'" + order_qt_utc_east "SELECT seq, created FROM typed_records" + sql "SET time_zone = '-05:00'" + order_qt_utc_west "SELECT seq, created FROM typed_records" + sql "SET time_zone = '${originalTimeZone}'" + order_qt_where "SELECT seq, name FROM typed_records WHERE score >= 10 AND active = true" + order_qt_or_in "SELECT seq FROM typed_records WHERE grp IN ('east', 'west') AND (score IS NULL OR score < 20)" + order_qt_like "SELECT seq, name FROM typed_records WHERE name LIKE '%a%'" + order_qt_regexp "SELECT seq, name FROM typed_records WHERE name REGEXP '^(alpha|beta)\$'" + order_qt_unicode "SELECT seq FROM typed_records WHERE name LIKE '%中文%'" + order_qt_group "SELECT grp, count(*), sum(score), min(seq), max(seq) FROM typed_records GROUP BY grp" + order_qt_join """SELECT a.seq, b.seq FROM typed_records a JOIN typed_records b + ON a.grp = b.grp WHERE a.seq < b.seq""" + order_qt_array "SELECT seq, array_size(tags), array_contains(tags, 'red') FROM typed_records" + order_qt_window "SELECT seq, row_number() OVER (PARTITION BY grp ORDER BY seq) FROM typed_records" + qt_order_limit "SELECT seq, name FROM typed_records ORDER BY seq DESC LIMIT 2 OFFSET 1" + order_qt_count "SELECT count(*) FROM typed_records" + order_qt_empty "SELECT count(*), count(seq) FROM empty_records" + order_qt_zero_match "SELECT seq FROM typed_records WHERE seq > 100" + order_qt_multi_batch "SELECT count(*), min(seq), max(seq), sum(seq) FROM large_records" + order_qt_batch_edges "SELECT seq FROM large_records WHERE seq IN (0,126,127,128,1023,1024,1025,2047,2048,2080)" + qt_batch_offset "SELECT seq FROM large_records ORDER BY seq LIMIT 5 OFFSET 1022" + for (int i = 0; i < 3; ++i) { + "qt_repeated_limit_${i}" "SELECT seq FROM large_records ORDER BY seq LIMIT 3" + "qt_early_cursor_close_${i}" "SELECT count(*) FROM (SELECT seq FROM large_records LIMIT 1) limited_rows" + } + order_qt_after_limits "SELECT count(*) FROM large_records" + List fallbackValidators = ["tuple_validator", "pattern_validator", "dependency_validator", + "nested_pattern_validator"] + for (String name : fallbackValidators) { + "order_qt_fallback_schema_${name}" "DESC ${name}" + "order_qt_fallback_values_${name}" "SELECT cast(_mongo_ejson AS STRING) FROM ${name}" + } + order_qt_fallback_schema "DESC unsupported_validator" + order_qt_fallback "SELECT cast(_mongo_ejson AS STRING), length(_mongo_bson) > 0 FROM unsupported_validator" + test { + sql "SELECT seq FROM historical_records" + exception "BSON" + } + order_qt_view_document "SELECT cast(_mongo_ejson AS STRING) FROM record_view" + order_qt_timeseries_document "SELECT count(*) FROM measurements" + + create("mongodb_native_documents", ["mongodb.schema.mode": "document"]) + sql "SWITCH mongodb_native_documents" + sql "USE doris_mongodb_native" + order_qt_document_override "DESC typed_records" + order_qt_document_bytes "SELECT count(*), min(length(_mongo_bson)), max(length(_mongo_bson)) FROM typed_records" + order_qt_extended_bson "SELECT cast(_mongo_ejson AS STRING) FROM special_records" + test { + sql "SELECT _mongo_ejson FROM deep_limit_records" + exception "BSON document exceeds the nesting limit" + } + create("mongodb_native_validators", ["mongodb.schema.mode": "validator"]) + sql "SWITCH mongodb_native_validators" + sql "USE doris_mongodb_native" + order_qt_forced_validator "SELECT count(*), sum(seq) FROM typed_records" + for (String object : ["record_view", "measurements", "unsupported_validator"]) { + test { + sql "SELECT * FROM ${object}" + exception "validator" + } + } + + for (String name : fallbackValidators) { + test { + sql "SELECT * FROM ${name}" + exception(name == "tuple_validator" ? "MongoDB array requires homogeneous items" + : "MongoDB schema constructs require an explicit mapping") + } + } + + def column = { String name, String type, List path = null, List tags = [] -> + [name: name, path: path == null ? [name] : path, doris_type: type, bson_types: tags] + } + Map manifest = [ + "doris_mongodb_native.record_view": [columns: [column("seq", "BIGINT", null, ["long"]), + column("label", "STRING", null, ["string"])]], + "doris_mongodb_native.measurements": [columns: [column("observed", "DATETIME(3)", null, ["date"]), + column("device", "STRING", null, ["string"]), column("value", "INT", null, ["int"])]], + "doris_mongodb_native.special_records": [columns: [ + column("identifier", "VARBINARY(12)", ["_id"], ["objectId"]), + column("integral", "BIGINT", null, ["long"]), column("exact_amount", "DECIMAL(10,4)", ["decimal"], ["decimal"]), + column("literal", "STRING", ["literal.dot"], ["string"]), + column("nested", "STRING", ["nested", "dot"], ["string"]), + column("dollar", "STRING", ['dollar$field'], ["string"]), + column("pattern", "JSON", null, ["regex"]), column("low", "JSON", null, ["minKey"]), + column("high", "JSON", null, ["maxKey"]), column("code", "JSON", null, ["javascriptWithScope"]), + column("refs", "ARRAY", null, ["array"]) + ]] + ] + manifest["doris_mongodb_native.wide_records"] = [columns: [column("seq", "INT", null, ["int"])]] + List boundaryColumns = [column("seq", "INT", null, ["int"]), + column("bool_v", "BOOLEAN", null, ["bool"]), column("tiny_v", "TINYINT", null, ["int"]), + column("small_v", "SMALLINT", null, ["int"]), column("int_v", "INT", null, ["int", "long"]), + column("long_v", "BIGINT", null, ["long"]), column("large_v", "LARGEINT", null, ["long"]), + column("float_v", "FLOAT", null, ["double"]), column("double_v", "DOUBLE", null, ["double"]), + column("text_v", "STRING", null, ["string"]), + column("oid_v", "VARBINARY(4)", null, ["objectId"]), + column("bin_v", "VARBINARY(1)", null, ["binData"]) + [subtype: 128], + column("dec32_v", "DECIMAL(9,2)", null, ["decimal"]), + column("dec64_v", "DECIMAL(18,4)", null, ["decimal"]), + column("dec128_v", "DECIMAL(38,4)", null, ["decimal"]), + column("date_v", "DATETIME(3)", null, ["date"]), + column("date_ms", "BIGINT", ["date_v"], ["date"]), + column("date_precision_v", "DATETIME(0)", null, ["date"]), + column("timestamp_v", "STRUCT", ["timestamp_v", "value"], ["timestamp"]), + column("regex_v", "STRUCT", null, ["regex"]), + column("array_v", "ARRAY", null, ["array"]), + column("object_v", "STRUCT", null, ["object"]), + column("nested_v", "ARRAY>", null, ["array"]), + column("json_v", "JSON"), column("decimal_json_v", "JSON", null, ["decimal"]), + column("min_key_v", "JSON", null, ["minKey"]), column("max_key_v", "JSON", null, ["maxKey"]), + column("code_v", "JSON", null, ["javascript"]), column("scope_v", "JSON", null, ["javascriptWithScope"])] + manifest["doris_mongodb_native.boundary_records"] = [columns: boundaryColumns] + manifest["doris_mongodb_native.boundary_errors"] = [columns: boundaryColumns] + create("mongodb_native_explicit", ["mongodb.schema.mode": "explicit", "mongodb.schema": JsonOutput.toJson(manifest)]) + sql "SWITCH mongodb_native_explicit" + sql "USE doris_mongodb_native" + order_qt_wide_scalar "SELECT seq FROM wide_records" + order_qt_wide_count "SELECT count(*) FROM wide_records WHERE seq = 1" + order_qt_view "SELECT seq, label FROM record_view" + order_qt_view_analysis "SELECT count(*), min(seq), max(seq) FROM record_view WHERE label LIKE 'view-%'" + order_qt_timeseries "SELECT observed, device, value FROM measurements" + order_qt_timeseries_where """SELECT device, count(*), sum(value) FROM measurements + WHERE observed >= '2024-01-01 00:00:00' AND observed < '2024-01-02 00:00:00' GROUP BY device""" + order_qt_explicit_paths "SELECT from_binary(identifier), integral, exact_amount, literal, nested, dollar FROM special_records" + order_qt_special_types "SELECT cast(pattern AS STRING), cast(low AS STRING), cast(high AS STRING), cast(code AS STRING) FROM special_records" + order_qt_refs "SELECT array_size(refs), from_binary(refs[1]), refs[2] IS NULL FROM special_records" + + String nullChecks = boundaryColumns.findAll { it.name != "seq" } + .collect { "${it.name} IS NULL" }.join(", ") + order_qt_type_nulls "SELECT seq, ${nullChecks} FROM boundary_records WHERE seq IN (1, 2)" + order_qt_integer_decimal_bounds """SELECT seq, bool_v, tiny_v, small_v, int_v, long_v, large_v, + dec32_v, dec64_v, dec128_v FROM boundary_records WHERE seq IN (3, 4, 5, 10)""" + order_qt_float_bounds """SELECT seq, cast(float_v AS STRING), cast(double_v AS STRING) + FROM boundary_records WHERE seq >= 3""" + order_qt_binary_string_bounds """SELECT seq, from_binary(oid_v), from_binary(bin_v), length(bin_v), + length(text_v), char_length(text_v), hex(substring(text_v, 1, 8)) + FROM boundary_records WHERE seq IN (3, 4, 5)""" + order_qt_datetime_timestamp_bounds """SELECT seq, cast(date_v AS STRING), date_ms, cast(date_precision_v AS STRING), + timestamp_v.seconds, timestamp_v.increment FROM boundary_records WHERE seq IN (3, 4, 5, 10)""" + order_qt_complex_bounds """SELECT seq, array_v, object_v, nested_v, cast(json_v AS STRING) + FROM boundary_records WHERE seq IN (3, 4, 5)""" + order_qt_special_bounds """SELECT seq, regex_v.pattern, regex_v.options, cast(min_key_v AS STRING), + cast(max_key_v AS STRING), cast(code_v AS STRING), cast(scope_v AS STRING), + cast(decimal_json_v AS STRING) FROM boundary_records WHERE seq >= 3""" + order_qt_epoch_outside_datetime_range "SELECT seq, date_ms FROM boundary_errors WHERE date_ms IS NOT NULL" + List errorFields = ["bool_v", "tiny_v", "small_v", "int_v", "long_v", "large_v", "float_v", + "text_v", "oid_v", "bin_v", "dec32_v", "dec64_v", "dec128_v", "date_v", "date_precision_v", + "timestamp_v", "regex_v", "array_v", "object_v", "nested_v"] + for (String field : errorFields) { + test { + sql "SELECT ${field} FROM boundary_errors" + exception "BSON" + } + } + + manifest["doris_mongodb_native.special_records"].columns.find { it.name == "literal" }.path = ["nested", "dot"] + String changedSchema = JsonOutput.toJson(manifest).replace("\\", "\\\\").replace("'", "\\'") + sql """ALTER CATALOG mongodb_native_explicit SET PROPERTIES ('mongodb.schema'='${changedSchema}')""" + sql "REFRESH CATALOG mongodb_native_explicit" + order_qt_refresh "SELECT literal, nested FROM special_records" + + List mismatchColumns = [column("seq", "BIGINT"), column("n", "BIGINT"), column("arr", "ARRAY"), + column("obj", "STRUCT"), column("amount", "DECIMAL(10,2)")] + Map mismatchManifest = ["doris_mongodb_native.mismatched": [columns: mismatchColumns]] + mismatchManifest["doris_mongodb_native.boundary_errors"] = [columns: boundaryColumns] + mismatchManifest["doris_mongodb_native.deep_limit_records"] = [columns: [column("value", "JSON")]] + create("mongodb_native_nulls", ["mongodb.schema.mode": "explicit", "mongodb.type_mismatch": "null", + "mongodb.schema": JsonOutput.toJson(mismatchManifest)]) + sql "SWITCH mongodb_native_nulls" + sql "USE doris_mongodb_native" + order_qt_mismatch_nulls "SELECT seq, n, arr, obj, amount FROM mismatched" + order_qt_mismatch_filter "SELECT seq FROM mismatched WHERE n IS NULL OR arr IS NULL OR obj IS NULL OR amount IS NULL" + String errorNullChecks = errorFields.collect { "${it} IS NULL" }.join(", ") + order_qt_boundary_error_nulls "SELECT seq, ${errorNullChecks} FROM boundary_errors" + test { + sql "SELECT value FROM deep_limit_records" + exception "BSON document exceeds the nesting limit" + } + create("mongodb_native_errors", ["mongodb.schema.mode": "explicit", + "mongodb.schema": JsonOutput.toJson(mismatchManifest)]) + sql "SWITCH mongodb_native_errors" + sql "USE doris_mongodb_native" + for (String field : ["n", "arr", "obj", "amount"]) { + test { + sql "SELECT ${field} FROM mismatched" + exception "BSON" + } + } + create("mongodb_native_view_only", ["mongodb.username": "native_view_reader", + "mongodb.password": "native_view_reader", "mongodb.auth_database": "doris_mongodb_native", + "mongodb.schema": JsonOutput.toJson(manifest)]) + sql "SWITCH mongodb_native_view_only" + sql "USE doris_mongodb_native" + order_qt_view_only_discovery "SHOW TABLES" + order_qt_view_only_read "SELECT seq, label FROM record_view" + create("mongodb_native_pagination", ["mongodb.databases": "doris_mongodb_pagination"]) + sql "SWITCH mongodb_native_pagination" + sql "USE doris_mongodb_pagination" + order_qt_metadata_pagination "SHOW TABLES LIKE 'page_13%'" + sql "SWITCH internal" +} diff --git a/run-fe-ut.sh b/run-fe-ut.sh index 6fd609121cd463..dcb8c551c315bf 100755 --- a/run-fe-ut.sh +++ b/run-fe-ut.sh @@ -204,6 +204,7 @@ FE_MODULES+=("fe-connector/fe-connector-adbc") # dependency (for Legacy413ProviderTypeContractTest) and -am pulls them into the reactor; fluss is # not among them, so it has to be named here or its tests never run. FE_MODULES+=("fe-connector/fe-connector-fluss") +FE_MODULES+=("fe-connector/fe-connector-mongodb") # The BE Java plugin modules. Nothing else runs these tests: no be-java-extensions module is # upstream of fe-core, so -am never reaches one, build.sh builds the reactor with -DskipTests, and # no GitHub workflow mentions the directory at all. What is in there is the evidence that the