From 7cba466547ffc7f4dd81ae6b7923fe6da011239c Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 24 Oct 2026 21:13:47 +0200 Subject: [PATCH] Backport of 117106 support dynamic type in parquet reader --- src/Core/FormatFactorySettings.h | 3 + src/Core/SettingsChangesHistory.cpp | 1 + src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 1 + .../Formats/Impl/Parquet/Reader.cpp | 18 +- .../Formats/Impl/Parquet/SchemaConverter.cpp | 147 +++- .../Formats/Impl/Parquet/SchemaConverter.h | 1 + .../Formats/Impl/Parquet/VariantEncoding.cpp | 816 ++++++++++++++++++ .../Formats/Impl/Parquet/VariantEncoding.h | 24 + .../Impl/ParquetV3BlockInputFormat.cpp | 11 + .../0_stateless/00900_long_parquet_load_2.sh | 19 + .../04928_parquet_variant_read.reference | 13 + .../0_stateless/04928_parquet_variant_read.sh | 37 + .../04929_parquet_variant_all_types.reference | 27 + .../04929_parquet_variant_all_types.sh | 18 + .../04930_parquet_variant_shredded.reference | 8 + .../04930_parquet_variant_shredded.sh | 34 + ...arquet_variant_max_dynamic_types.reference | 73 ++ ...04931_parquet_variant_max_dynamic_types.sh | 29 + ..._parquet_variant_null_as_default.reference | 8 + .../04932_parquet_variant_null_as_default.sh | 21 + ...arquet_variant_binary_and_string.reference | 6 + ...04933_parquet_variant_binary_and_string.sh | 23 + ...227_parquet_variant_nested_types.reference | 11 + .../05227_parquet_variant_nested_types.sh | 27 + ...8_parquet_variant_malformed_blob.reference | 5 + .../05228_parquet_variant_malformed_blob.sh | 16 + ...296_parquet_variant_json_objects.reference | 13 + .../05296_parquet_variant_json_objects.sh | 25 + ...parquet_variant_json_shared_data.reference | 1 + .../05297_parquet_variant_json_shared_data.sh | 22 + ...quet_variant_json_duplicate_path.reference | 1 + ...298_parquet_variant_json_duplicate_path.sh | 13 + ...299_parquet_variant_read_as_json.reference | 52 ++ .../05299_parquet_variant_read_as_json.sh | 64 ++ ...quet_variant_unsupported_version.reference | 2 + ...306_parquet_variant_unsupported_version.sh | 17 + ...7_parquet_variant_one_sided_null.reference | 4 + .../05307_parquet_variant_one_sided_null.sh | 24 + .../data_parquet/04928_variant_spark.parquet | Bin 0 -> 921 bytes .../04929_variant_all_types.parquet | Bin 0 -> 1476 bytes .../04930_variant_shredded.parquet | Bin 0 -> 1238 bytes .../04931_variant_max_dynamic_types.parquet | Bin 0 -> 1217 bytes .../data_parquet/04932_variant_null.parquet | Bin 0 -> 724 bytes .../04933_variant_binary_and_string.parquet | Bin 0 -> 802 bytes .../05227_variant_nested_types.parquet | Bin 0 -> 1653 bytes .../05228_variant_malformed.parquet | Bin 0 -> 1001 bytes .../05296_variant_json_objects.parquet | Bin 0 -> 1568 bytes .../05297_variant_json_shared_data.parquet | Bin 0 -> 24918 bytes .../05298_variant_json_duplicate_path.parquet | Bin 0 -> 1378 bytes .../05299_variant_read_as_json.parquet | Bin 0 -> 1212 bytes .../05306_variant_unsupported_version.parquet | Bin 0 -> 1568 bytes .../05307_variant_one_sided_null.parquet | Bin 0 -> 977 bytes 53 files changed, 1632 insertions(+), 4 deletions(-) create mode 100644 src/Processors/Formats/Impl/Parquet/VariantEncoding.cpp create mode 100644 src/Processors/Formats/Impl/Parquet/VariantEncoding.h create mode 100644 tests/queries/0_stateless/04928_parquet_variant_read.reference create mode 100755 tests/queries/0_stateless/04928_parquet_variant_read.sh create mode 100644 tests/queries/0_stateless/04929_parquet_variant_all_types.reference create mode 100755 tests/queries/0_stateless/04929_parquet_variant_all_types.sh create mode 100644 tests/queries/0_stateless/04930_parquet_variant_shredded.reference create mode 100755 tests/queries/0_stateless/04930_parquet_variant_shredded.sh create mode 100644 tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.reference create mode 100755 tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.sh create mode 100644 tests/queries/0_stateless/04932_parquet_variant_null_as_default.reference create mode 100755 tests/queries/0_stateless/04932_parquet_variant_null_as_default.sh create mode 100644 tests/queries/0_stateless/04933_parquet_variant_binary_and_string.reference create mode 100755 tests/queries/0_stateless/04933_parquet_variant_binary_and_string.sh create mode 100644 tests/queries/0_stateless/05227_parquet_variant_nested_types.reference create mode 100755 tests/queries/0_stateless/05227_parquet_variant_nested_types.sh create mode 100644 tests/queries/0_stateless/05228_parquet_variant_malformed_blob.reference create mode 100755 tests/queries/0_stateless/05228_parquet_variant_malformed_blob.sh create mode 100644 tests/queries/0_stateless/05296_parquet_variant_json_objects.reference create mode 100755 tests/queries/0_stateless/05296_parquet_variant_json_objects.sh create mode 100644 tests/queries/0_stateless/05297_parquet_variant_json_shared_data.reference create mode 100755 tests/queries/0_stateless/05297_parquet_variant_json_shared_data.sh create mode 100644 tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.reference create mode 100755 tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.sh create mode 100644 tests/queries/0_stateless/05299_parquet_variant_read_as_json.reference create mode 100755 tests/queries/0_stateless/05299_parquet_variant_read_as_json.sh create mode 100644 tests/queries/0_stateless/05306_parquet_variant_unsupported_version.reference create mode 100755 tests/queries/0_stateless/05306_parquet_variant_unsupported_version.sh create mode 100644 tests/queries/0_stateless/05307_parquet_variant_one_sided_null.reference create mode 100755 tests/queries/0_stateless/05307_parquet_variant_one_sided_null.sh create mode 100644 tests/queries/0_stateless/data_parquet/04928_variant_spark.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04929_variant_all_types.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04930_variant_shredded.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04931_variant_max_dynamic_types.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04932_variant_null.parquet create mode 100644 tests/queries/0_stateless/data_parquet/04933_variant_binary_and_string.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05227_variant_nested_types.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05228_variant_malformed.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05296_variant_json_objects.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05297_variant_json_shared_data.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05298_variant_json_duplicate_path.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05299_variant_read_as_json.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05306_variant_unsupported_version.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05307_variant_one_sided_null.parquet diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index 12fad80b961bf..32f00244c633c 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1711,6 +1711,9 @@ Indicate which field of protobuf oneof was found by means of setting enum value )", 0) \ DECLARE(Bool, input_format_parquet_allow_geoparquet_parser, true, R"( Use geo column parser to convert Array(UInt8) into Point/MultiPoint/Linestring/Polygon/MultiLineString/MultiPolygon types +)", 0) \ + DECLARE(Bool, input_format_parquet_detect_variant_by_structure, true, R"( +Read a Parquet group that has no `VARIANT` logical type, but has the layout of an unshredded variant (a `metadata` and a `value` field of type `BYTE_ARRAY`), as a variant, i.e. as `Dynamic`, instead of `Tuple`. Spark 4.0 writes variant columns this way. Groups annotated with the `VARIANT` logical type are always read as a variant. )", 0) \ DECLARE(Bool, output_format_parquet_geometadata, true, R"( Allow to write information about geo columns in parquet metadata and encode columns in WKB format. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 6e44ec5efbc01..cb7466bfed1d6 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.8", { + {"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 34c81d515589a..9afd619025a24 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -264,6 +264,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.verify_checksums = settings[Setting::input_format_parquet_verify_checksums]; format_settings.parquet.local_time_as_utc = settings[Setting::input_format_parquet_local_time_as_utc]; format_settings.parquet.allow_geoparquet_parser = settings[Setting::input_format_parquet_allow_geoparquet_parser]; + format_settings.parquet.detect_variant_by_structure = settings[Setting::input_format_parquet_detect_variant_by_structure]; format_settings.parquet.write_geometadata = settings[Setting::output_format_parquet_geometadata]; if (auto memory_limit = total_memory_tracker.getHardLimit(); memory_limit > 0) { diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index dfd95af1fd981..69c6676914c3f 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -410,6 +410,7 @@ struct FormatSettings double bloom_filter_bits_per_value = 10.5; size_t bloom_filter_flush_threshold_bytes = 1024 * 1024 * 128; bool allow_geoparquet_parser = true; + bool detect_variant_by_structure = true; bool spatial_filter_push_down = true; bool write_geometadata = true; size_t max_dictionary_size = 1024 * 1024; diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index e92a183b0d79b..8d69ee81852c8 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -1,12 +1,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -21,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -3280,9 +3283,8 @@ MutableColumnPtr Reader::formOutputColumn(RowSubgroup & row_subgroup, size_t out nullable_group_null_map = ColumnUInt8::create(num_rows, UInt8(0)); } - TypeIndex kind = output_info.nullable_group - ? removeNullable(output_info.input_type)->getColumnType() - : output_info.input_type->getColumnType(); + /// Nullable wraps the type of a physically nullable tuple group and of a variant read as `Nullable(JSON)`. + TypeIndex kind = removeNullable(output_info.input_type)->getColumnType(); if (output_info.is_primitive) { @@ -3334,6 +3336,16 @@ MutableColumnPtr Reader::formOutputColumn(RowSubgroup & row_subgroup, size_t out else res = ColumnTuple::create(std::move(columns)); } + else if (kind == TypeIndex::Dynamic || kind == TypeIndex::Object) + { + chassert(output_info.nested_columns.size() == 2); + MutableColumnPtr metadata = formOutputColumn(row_subgroup, output_info.nested_columns[0], num_rows); + MutableColumnPtr value = formOutputColumn(row_subgroup, output_info.nested_columns[1], num_rows); + + res = output_info.input_type->createColumn(); + res->reserve(num_rows); + decodeVariantColumn(*metadata, *value, *res, output_info.input_type, output_info.name, num_rows, options.format); + } else { chassert(kind == TypeIndex::Map); diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index 3723016eb09d8..97df3cea5d11c 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -295,7 +296,8 @@ void SchemaConverter::processSubtree(TraversalNode & node) if (!processSubtreePrimitive(node) && !processSubtreeMap(node) && !processSubtreeArrayOuter(node) && - !processSubtreeArrayInner(node)) + !processSubtreeArrayInner(node) && + !processSubtreeDynamic(node)) { processSubtreeTuple(node); } @@ -667,6 +669,149 @@ static bool tupleSubtreeIsAllRequired(const std::vector & s return true; } +static std::optional subtreeEnd(const std::vector & schema, size_t idx) +{ + size_t remaining = 1; + while (remaining > 0) + { + if (idx >= schema.size() || remaining > schema.size()) + return std::nullopt; + const parq::SchemaElement & element = schema[idx]; + remaining -= 1; + if (!isPrimitiveNode(element)) + { + if (element.num_children < 0 || size_t(element.num_children) > schema.size()) + return std::nullopt; + remaining += size_t(element.num_children); + } + idx += 1; + } + return idx; +} + +bool SchemaConverter::processSubtreeDynamic(TraversalNode & node) +{ + if (node.element->num_children != 2 && node.element->num_children != 3) + return false; + const size_t num_children = size_t(node.element->num_children); + if (num_children > file_metadata.schema.size() - schema_idx) + return false; + + const bool group_annotated_variant = node.element->logicalType.__isset.VARIANT; + + enum ChildRole : size_t { Metadata = 0, Value = 1, TypedValue = 2, NumRoles = 3 }; + + std::array, NumRoles> schema_idx_of_role; + std::vector role_of_child(num_children); + size_t child_schema_idx = schema_idx; + for (size_t i = 0; i < num_children; ++i) + { + const parq::SchemaElement & child = file_metadata.schema[child_schema_idx]; + size_t role = 0; + if (child.name == "metadata") + role = Metadata; + else if (child.name == "value") + role = Value; + else if (child.name == "typed_value") + role = TypedValue; + else + return false; + + if (schema_idx_of_role[role].has_value()) + return false; + schema_idx_of_role[role] = child_schema_idx; + role_of_child[i] = role; + + std::optional end = subtreeEnd(file_metadata.schema, child_schema_idx); + if (!end.has_value()) + return false; + child_schema_idx = *end; + } + + if (!schema_idx_of_role[Metadata].has_value()) + return false; + if (!schema_idx_of_role[Value].has_value() && !schema_idx_of_role[TypedValue].has_value()) + return false; + + /// `metadata` and `value` hold variant-encoded blobs: unannotated BYTE_ARRAY, not an array. + auto is_variant_blob_leaf = [&](size_t idx) + { + const parq::SchemaElement & element = file_metadata.schema[idx]; + return isPrimitiveNode(element) + && element.type == parq::Type::BYTE_ARRAY + && element.repetition_type != parq::FieldRepetitionType::REPEATED + && (group_annotated_variant || (!element.logicalType.__isset.STRING && !element.__isset.converted_type)); + }; + + if (!is_variant_blob_leaf(*schema_idx_of_role[Metadata])) + return false; + if (schema_idx_of_role[Value].has_value() && !is_variant_blob_leaf(*schema_idx_of_role[Value])) + return false; + + const DataTypePtr type_hint = node.type_hint ? removeNullable(node.type_hint) : nullptr; + const bool read_as_json = type_hint && isObject(type_hint); + if (type_hint && !read_as_json && !isDynamic(type_hint)) + return false; + + /// Spark 4.0 writes variant columns without the `VARIANT` logical type, so they can only be + /// recognized by structure. Before this was supported, such groups were read as `Tuple`, hence + /// the setting. An explicit `Dynamic`/`JSON` request is not ambiguous and bypasses it. + if (!group_annotated_variant && !node.type_hint && !options.format.parquet.detect_variant_by_structure) + return false; + + std::optional unsupported; + if (group_annotated_variant && node.element->logicalType.VARIANT.__isset.specification_version + && node.element->logicalType.VARIANT.specification_version != 1) + unsupported = fmt::format( + "a variant with specification version {}, but only version 1 is supported", + Int16(node.element->logicalType.VARIANT.specification_version)); + else if (schema_idx_of_role[TypedValue].has_value()) + unsupported = "a shredded variant (it has a `typed_value` field), which is not supported yet"; + + size_t primitive_start = primitive_columns.size(); + size_t output_start = output_columns.size(); + + /// Recurse even when the column is unsupported or not requested, to advance the traversal + /// past the group's children. + std::array, NumRoles> output_idx_of_role; + for (size_t i = 0; i < num_children; ++i) + { + TraversalNode subnode = node.prepareToRecurse(SchemaContext::None, nullptr); + subnode.requested = node.requested && !unsupported.has_value(); + processSubtree(subnode); + output_idx_of_role[role_of_child[i]] = subnode.output_idx; + } + + if (!node.requested) + return true; + + if (unsupported.has_value()) + { + /// Like for unsupported primitive types, schema inference can skip the column. When reading, + /// a requested column must not silently come out empty. + if (!sample_block && options.format.parquet.skip_columns_with_unsupported_types_in_schema_inference) + return true; + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Parquet column {} is {}", node.getNameForLogging(), *unsupported); + } + + if (!output_idx_of_role[Metadata].has_value() || !output_idx_of_role[Value].has_value()) + { + primitive_columns.resize(primitive_start); + output_columns.resize(output_start); + return true; + } + + node.output_idx = output_columns.size(); + OutputColumnInfo & output = output_columns.emplace_back(); + output.name = node.name; + output.primitive_start = primitive_start; + output.primitive_end = primitive_columns.size(); + output.input_type = read_as_json ? node.type_hint : std::make_shared(); + output.output_type = output.input_type; + output.nested_columns = {output_idx_of_role[Metadata].value(), output_idx_of_role[Value].value()}; + return true; +} + void SchemaConverter::processSubtreeTuple(TraversalNode & node) { /// Tuple (possibly a Map key_value tuple): diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h index 6d4efafcd4512..edb85dff246eb 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h @@ -144,6 +144,7 @@ struct SchemaConverter bool processSubtreeArrayOuter(TraversalNode & node); bool processSubtreeArrayInner(TraversalNode & node); void processSubtreeTuple(TraversalNode & node); + bool processSubtreeDynamic(TraversalNode & node); void processPrimitiveColumn( const parq::SchemaElement & element, DataTypePtr type_hint, diff --git a/src/Processors/Formats/Impl/Parquet/VariantEncoding.cpp b/src/Processors/Formats/Impl/Parquet/VariantEncoding.cpp new file mode 100644 index 0000000000000..892a0633177a8 --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/VariantEncoding.cpp @@ -0,0 +1,816 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int TOO_DEEP_RECURSION; + extern const int TYPE_MISMATCH; +} + +namespace DB::Parquet +{ + +namespace +{ + +/// https://github.com/apache/parquet-format/blob/master/VariantEncoding.md +enum class PrimitiveType : UInt8 +{ + Null = 0, + True = 1, + False = 2, + Int8 = 3, + Int16 = 4, + Int32 = 5, + Int64 = 6, + Double = 7, + Decimal4 = 8, + Decimal8 = 9, + Decimal16 = 10, + Date = 11, + TimestampTZ = 12, + TimestampNTZ = 13, + Float = 14, + Binary = 15, + String = 16, + TimeNTZ = 17, + TimestampNanosTZ = 18, + TimestampNanosNTZ = 19, + UUID = 20 +}; + +enum class BasicType : UInt8 +{ + Primitive = 0, + ShortString = 1, + Object = 2, + Array = 3 +}; + +void checkRange(std::string_view data, size_t pos, size_t size) +{ + if (pos > data.size() || size > data.size() - pos) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed Parquet variant: {} bytes are needed at offset {}, but the blob is {} bytes", + size, pos, data.size()); +} + +template +T readFixed(std::string_view data, size_t pos) +{ + checkRange(data, pos, sizeof(T)); + return unalignedLoadLittleEndian(data.data() + pos); +} + +UInt8 readByte(std::string_view data, size_t pos) +{ + checkRange(data, pos, 1); + return UInt8(data[pos]); +} + +std::string_view readSlice(std::string_view data, size_t pos, size_t size) +{ + checkRange(data, pos, size); + return data.substr(pos, size); +} + +UInt32 readUnsigned(std::string_view data, size_t pos, UInt8 size) +{ + checkRange(data, pos, size); + UInt32 res = 0; + for (UInt8 i = 0; i < size; ++i) + res |= UInt32(UInt8(data[pos + i])) << (8 * i); + return res; +} + +struct Metadata +{ + std::string_view blob; + size_t offsets_pos = 0; + size_t names_pos = 0; + UInt32 count = 0; + UInt8 offset_size = 0; + + std::string_view getName(UInt32 id) const + { + if (id >= count) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed Parquet variant: object field refers to dictionary entry {}, but the metadata " + "dictionary has {} entries", id, count); + const UInt32 begin = readUnsigned(blob, offsets_pos + size_t(id) * offset_size, offset_size); + const UInt32 end = readUnsigned(blob, offsets_pos + (size_t(id) + 1) * offset_size, offset_size); + if (end < begin) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed Parquet variant: metadata dictionary entry {} ends at {}, before its start {}", + id, end, begin); + return readSlice(blob, names_pos + begin, end - begin); + } +}; + +Metadata parseMetadata(std::string_view blob) +{ + const UInt8 header = readByte(blob, 0); + const UInt8 version = header & 0x0F; + if (version != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unsupported Parquet variant metadata version {}", UInt16(version)); + + Metadata res; + res.blob = blob; + res.offset_size = ((header >> 6) & 0x03) + 1; + res.count = readUnsigned(blob, 1, res.offset_size); + res.offsets_pos = 1 + res.offset_size; + res.names_pos = res.offsets_pos + (size_t(res.count) + 1) * res.offset_size; + checkRange(blob, res.offsets_pos, res.names_pos - res.offsets_pos); + return res; +} + +struct DecodeContext +{ + const Metadata & metadata; + size_t max_depth; +}; + +void checkDepth(const DecodeContext & context, size_t depth) +{ + checkStackSize(); + if (context.max_depth != 0 && depth > context.max_depth) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Parquet variant value is nested deeper than the limit ({}). It can be raised with the " + "setting 'max_parser_depth', but a very deeply nested value is rarely intentional", + context.max_depth); +} + +/// Typed paths and `SKIP` clauses of a `JSON` column requested by the user. +class JSONPathsFilter +{ +public: + JSONPathsFilter(const DataTypeObject & type_, bool use_partial_match_) + : type(type_) + , sorted_paths_to_skip(type.getPathsToSkip().begin(), type.getPathsToSkip().end()) + , use_partial_match(use_partial_match_) + { + std::sort(sorted_paths_to_skip.begin(), sorted_paths_to_skip.end()); + for (const auto & regexp : type.getPathRegexpsToSkip()) + path_regexps_to_skip.emplace_back(regexp); + } + + bool shouldSkip(const String & path) const + { + if (type.getPathsToSkip().contains(path)) + return true; + + auto it = std::lower_bound(sorted_paths_to_skip.begin(), sorted_paths_to_skip.end(), path); + if (it != sorted_paths_to_skip.begin() && path.starts_with(*std::prev(it) + ".")) + return true; + + for (const auto & regexp : path_regexps_to_skip) + { + if (use_partial_match ? re2::RE2::PartialMatch(path, regexp) : re2::RE2::FullMatch(path, regexp)) + return true; + } + return false; + } + + const DataTypePtr * findTypedPath(const String & path) const + { + const auto & typed_paths = type.getTypedPaths(); + auto it = typed_paths.find(path); + return it == typed_paths.end() ? nullptr : &it->second; + } + +private: + const DataTypeObject & type; + std::vector sorted_paths_to_skip; + std::list path_regexps_to_skip; + bool use_partial_match; +}; + +struct ObjectLayout +{ + UInt32 num_elements = 0; + UInt8 id_size = 0; + UInt8 offset_size = 0; + size_t ids_pos = 0; + size_t offsets_pos = 0; + size_t values_pos = 0; + + UInt32 fieldId(std::string_view data, UInt32 i) const + { + return readUnsigned(data, ids_pos + size_t(i) * id_size, id_size); + } + + size_t valuePos(std::string_view data, UInt32 i) const + { + return values_pos + readUnsigned(data, offsets_pos + size_t(i) * offset_size, offset_size); + } +}; + +ObjectLayout parseObjectLayout(std::string_view data, size_t pos, UInt8 value_header) +{ + const bool is_large = (value_header >> 4) & 0x01; + ObjectLayout res; + res.id_size = ((value_header >> 2) & 0x03) + 1; + res.offset_size = (value_header & 0x03) + 1; + res.num_elements = readUnsigned(data, pos, is_large ? 4 : 1); + res.ids_pos = pos + (is_large ? 4 : 1); + res.offsets_pos = res.ids_pos + size_t(res.num_elements) * res.id_size; + res.values_pos = res.offsets_pos + (size_t(res.num_elements) + 1) * res.offset_size; + checkRange(data, res.ids_pos, res.values_pos - res.ids_pos); + return res; +} + +struct ArrayLayout +{ + UInt32 num_elements = 0; + UInt8 offset_size = 0; + size_t offsets_pos = 0; + size_t values_pos = 0; + + size_t elementPos(std::string_view data, UInt32 i) const + { + return values_pos + readUnsigned(data, offsets_pos + size_t(i) * offset_size, offset_size); + } +}; + +ArrayLayout parseArrayLayout(std::string_view data, size_t pos, UInt8 value_header) +{ + const bool is_large = (value_header >> 2) & 0x01; + ArrayLayout res; + res.offset_size = (value_header & 0x03) + 1; + res.num_elements = readUnsigned(data, pos, is_large ? 4 : 1); + res.offsets_pos = pos + (is_large ? 4 : 1); + res.values_pos = res.offsets_pos + (size_t(res.num_elements) + 1) * res.offset_size; + checkRange(data, res.offsets_pos, res.values_pos - res.offsets_pos); + return res; +} + +bool isObject(std::string_view data, size_t pos) +{ + return BasicType(readByte(data, pos) & 0x03) == BasicType::Object; +} + +bool isArrayOfObjects(std::string_view data, size_t pos, UInt8 value_header) +{ + const ArrayLayout layout = parseArrayLayout(data, pos, value_header); + if (layout.num_elements == 0) + return false; + for (UInt32 i = 0; i < layout.num_elements; ++i) + { + if (!isObject(data, layout.elementPos(data, i))) + return false; + } + return true; +} + +String decimalTypeName(std::string_view data, size_t pos, UInt8 precision) +{ + const UInt8 scale = readByte(data, pos + 1); + if (scale > precision) + throw Exception( + ErrorCodes::INCORRECT_DATA, "Malformed Parquet variant: decimal with precision {} has invalid scale {}", UInt16(precision), UInt16(scale)); + return fmt::format("Decimal({}, {})", UInt16(precision), UInt16(scale)); +} + +/// The name of the type of the value at `pos`, without decoding it. Empty for a variant null. +String getValueTypeName(std::string_view data, size_t pos) +{ + const UInt8 header = readByte(data, pos); + const UInt8 value_header = header >> 2; + + switch (BasicType(header & 0x03)) + { + case BasicType::ShortString: + return "String"; + case BasicType::Object: + return "JSON"; + case BasicType::Array: + return isArrayOfObjects(data, pos + 1, value_header) ? "Array(JSON)" : "Array(Dynamic)"; + case BasicType::Primitive: + break; + } + + switch (PrimitiveType(value_header)) + { + case PrimitiveType::Null: + return {}; + case PrimitiveType::True: + case PrimitiveType::False: + return "Bool"; + case PrimitiveType::Int8: + return "Int8"; + case PrimitiveType::Int16: + return "Int16"; + case PrimitiveType::Int32: + return "Int32"; + case PrimitiveType::Int64: + return "Int64"; + case PrimitiveType::Float: + return "Float32"; + case PrimitiveType::Double: + return "Float64"; + case PrimitiveType::Decimal4: + return decimalTypeName(data, pos, 9); + case PrimitiveType::Decimal8: + return decimalTypeName(data, pos, 18); + case PrimitiveType::Decimal16: + return decimalTypeName(data, pos, 38); + case PrimitiveType::Date: + return "Date32"; + case PrimitiveType::TimestampTZ: + return "DateTime64(6, 'UTC')"; + case PrimitiveType::TimestampNTZ: + return "DateTime64(6)"; + case PrimitiveType::TimestampNanosTZ: + return "DateTime64(9, 'UTC')"; + case PrimitiveType::TimestampNanosNTZ: + return "DateTime64(9)"; + case PrimitiveType::TimeNTZ: + return "Time64(6)"; + case PrimitiveType::UUID: + return "UUID"; + case PrimitiveType::Binary: + case PrimitiveType::String: + return "String"; + } + + /// The id is 6 bits of a byte of the blob, so it can be any of 0..63, while the encoding spec + /// assigns only 0..20. + throw Exception( + ErrorCodes::INCORRECT_DATA, "Malformed Parquet variant: unknown primitive type id {}", UInt16(value_header)); +} + +void decodeValueIntoDynamic( + std::string_view data, size_t pos, const DecodeContext & context, size_t depth, ColumnDynamic & target); + +/// `target` must be a column of the type reported by getValueType for this value. +void decodeValueIntoColumn( + std::string_view data, size_t pos, const DecodeContext & context, size_t depth, IColumn & target); + +void decodePrimitiveIntoColumn(std::string_view data, size_t pos, PrimitiveType type_id, IColumn & target) +{ + switch (type_id) + { + case PrimitiveType::True: + assert_cast(target).insertValue(1); + return; + case PrimitiveType::False: + assert_cast(target).insertValue(0); + return; + case PrimitiveType::Int8: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Int16: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Int32: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Int64: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Float: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Double: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::Decimal4: + assert_cast &>(target).insertValue(Decimal32(readFixed(data, pos + 1))); + return; + case PrimitiveType::Decimal8: + assert_cast &>(target).insertValue(Decimal64(readFixed(data, pos + 1))); + return; + case PrimitiveType::Decimal16: + assert_cast &>(target).insertValue(Decimal128(readFixed(data, pos + 1))); + return; + case PrimitiveType::Date: + assert_cast(target).insertValue(readFixed(data, pos)); + return; + case PrimitiveType::TimestampTZ: + case PrimitiveType::TimestampNTZ: + case PrimitiveType::TimestampNanosTZ: + case PrimitiveType::TimestampNanosNTZ: + assert_cast &>(target).insertValue(DateTime64(readFixed(data, pos))); + return; + case PrimitiveType::TimeNTZ: + assert_cast &>(target).insertValue(Time64(readFixed(data, pos))); + return; + case PrimitiveType::UUID: + { + const std::string_view bytes = readSlice(data, pos, sizeof(UUID)); + UUID uuid; + memcpy(&uuid, bytes.data(), sizeof(UUID)); + auto * halves = reinterpret_cast(&uuid); + if constexpr (std::endian::native == std::endian::little) + { + std::reverse(halves, halves + 8); + std::reverse(halves + 8, halves + 16); + } + else + { + std::swap_ranges(halves, halves + 8, halves + 8); + } + assert_cast(target).insertValue(uuid); + return; + } + case PrimitiveType::Binary: + case PrimitiveType::String: + { + const UInt32 length = readFixed(data, pos); + const std::string_view value = readSlice(data, pos + 4, length); + assert_cast(target).insertData(value.data(), value.size()); + return; + } + case PrimitiveType::Null: + break; + } + + throw Exception( + ErrorCodes::INCORRECT_DATA, "Malformed Parquet variant: unknown primitive type id {}", UInt16(type_id)); +} + +void decodeValueIntoTypedPath( + std::string_view data, size_t pos, const DecodeContext & context, size_t depth, const DataTypePtr & typed_type, IColumn & target) +{ + const String type_name = getValueTypeName(data, pos); + if (type_name == typed_type->getName()) + { + decodeValueIntoColumn(data, pos, context, depth, target); + return; + } + + /// TODO: Add a fast path for simple conversions (number -> number, string -> number, + /// number -> string, etc.) that doesn't create a column and call castColumn: that is too + /// expensive to do per single value. + const DataTypePtr type = getDataTypesCache().getType(type_name); + auto value_column = type->createColumn(); + decodeValueIntoColumn(data, pos, context, depth, *value_column); + const ColumnPtr casted = castColumn({std::move(value_column), type, ""}, typed_type); + target.insertFrom(*casted, 0); +} + +struct SharedDataValue +{ + String path; + size_t pos; + size_t depth; +}; + +/// Nested objects are flattened into dot-separated paths, the same way the `JSON` type stores them. +void insertObjectPaths( + std::string_view data, + size_t pos, + UInt8 value_header, + const DecodeContext & context, + size_t depth, + const String & prefix, + bool is_root, + ColumnObject & column, + const JSONPathsFilter * filter, + size_t prev_size, + std::vector & shared_data_values) +{ + const ObjectLayout layout = parseObjectLayout(data, pos, value_header); + + for (UInt32 i = 0; i < layout.num_elements; ++i) + { + const std::string_view name = context.metadata.getName(layout.fieldId(data, i)); + String path = is_root ? String(name) : prefix + "." + String(name); + if (filter && filter->shouldSkip(path)) + continue; + + const size_t value_pos = layout.valuePos(data, i); + const UInt8 header = readByte(data, value_pos); + + /// Like in the `JSON` type, a null is equivalent to the absence of the path. + if (header == UInt8(PrimitiveType::Null) << 2) + continue; + + if (const DataTypePtr * typed_type = filter ? filter->findTypedPath(path) : nullptr) + { + auto & typed_column = *column.getTypedPaths().at(path); + if (typed_column.size() > prev_size) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet variant object has the path '{}' more than once after flattening nested objects", path); + decodeValueIntoTypedPath(data, value_pos, context, depth + 1, *typed_type, typed_column); + continue; + } + + if (BasicType(header & 0x03) == BasicType::Object) + { + checkDepth(context, depth + 1); + insertObjectPaths( + data, value_pos + 1, header >> 2, context, depth + 1, path, false, column, filter, prev_size, shared_data_values); + continue; + } + + ColumnDynamic * dynamic_column = nullptr; + auto & dynamic_paths = column.getDynamicPathsPtrs(); + if (auto it = dynamic_paths.find(path); it != dynamic_paths.end()) + { + dynamic_column = it->second; + if (dynamic_column->size() > prev_size) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet variant object has the path '{}' more than once after flattening nested objects", path); + } + else + { + dynamic_column = column.tryToAddNewDynamicPath(path); + } + + if (!dynamic_column) + { + shared_data_values.push_back({std::move(path), value_pos, depth + 1}); + continue; + } + + decodeValueIntoDynamic(data, value_pos, context, depth + 1, *dynamic_column); + } +} + +void decodeObjectIntoJSON( + std::string_view data, + size_t pos, + UInt8 value_header, + const DecodeContext & context, + size_t depth, + ColumnObject & column, + const JSONPathsFilter * filter = nullptr) +{ + const size_t prev_size = column.size(); + + std::vector shared_data_values; + insertObjectPaths(data, pos, value_header, context, depth, "", true, column, filter, prev_size, shared_data_values); + + /// Paths in shared data must be sorted. + std::sort( + shared_data_values.begin(), shared_data_values.end(), [](const auto & left, const auto & right) { return left.path < right.path; }); + + auto [shared_data_paths, shared_data_values_column] = column.getSharedDataPathsAndValues(); + MutableColumnPtr tmp_dynamic_column; + for (size_t i = 0; i < shared_data_values.size(); ++i) + { + const SharedDataValue & value = shared_data_values[i]; + if (i != 0 && value.path == shared_data_values[i - 1].path) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet variant object has the path '{}' more than once after flattening nested objects", value.path); + + if (!tmp_dynamic_column) + tmp_dynamic_column = ColumnDynamic::create(column.getMaxDynamicTypes()); + auto & tmp_dynamic = assert_cast(*tmp_dynamic_column); + /// Keep only the value being serialized, not all values of all shared data paths. + if (!tmp_dynamic.empty()) + tmp_dynamic.popBack(1); + decodeValueIntoDynamic(data, value.pos, context, value.depth, tmp_dynamic); + ColumnObject::serializePathAndValueIntoSharedData( + shared_data_paths, shared_data_values_column, value.path, tmp_dynamic, 0); + } + column.getSharedDataOffsets().push_back(shared_data_paths->size()); + + for (auto & [_, typed_column] : column.getTypedPaths()) + { + if (typed_column->size() == prev_size) + typed_column->insertDefault(); + } + + for (auto & [_, dynamic_column] : column.getDynamicPathsPtrs()) + { + if (dynamic_column->size() == prev_size) + dynamic_column->insertDefault(); + } +} + +void decodeArrayIntoColumn( + std::string_view data, size_t pos, UInt8 value_header, const DecodeContext & context, size_t depth, IColumn & target) +{ + const ArrayLayout layout = parseArrayLayout(data, pos, value_header); + + auto & array_column = assert_cast(target); + auto & elements = array_column.getData(); + + if (auto * objects = typeid_cast(&elements)) + { + for (UInt32 i = 0; i < layout.num_elements; ++i) + { + const size_t element_pos = layout.elementPos(data, i); + checkDepth(context, depth + 1); + decodeObjectIntoJSON(data, element_pos + 1, readByte(data, element_pos) >> 2, context, depth + 1, *objects); + } + } + else + { + auto & dynamic_elements = assert_cast(elements); + for (UInt32 i = 0; i < layout.num_elements; ++i) + decodeValueIntoDynamic(data, layout.elementPos(data, i), context, depth + 1, dynamic_elements); + } + + array_column.getOffsets().push_back(elements.size()); +} + +void decodeValueIntoColumn( + std::string_view data, size_t pos, const DecodeContext & context, size_t depth, IColumn & target) +{ + const UInt8 header = readByte(data, pos); + const UInt8 value_header = header >> 2; + + switch (BasicType(header & 0x03)) + { + case BasicType::Primitive: + decodePrimitiveIntoColumn(data, pos + 1, PrimitiveType(value_header), target); + return; + case BasicType::ShortString: + { + const std::string_view value = readSlice(data, pos + 1, value_header); + assert_cast(target).insertData(value.data(), value.size()); + return; + } + case BasicType::Object: + decodeObjectIntoJSON(data, pos + 1, value_header, context, depth, assert_cast(target)); + return; + case BasicType::Array: + decodeArrayIntoColumn(data, pos + 1, value_header, context, depth, target); + return; + } +} + +void decodeValueIntoDynamic( + std::string_view data, size_t pos, const DecodeContext & context, size_t depth, ColumnDynamic & target) +{ + checkDepth(context, depth); + + const String type_name = getValueTypeName(data, pos); + if (type_name.empty()) + { + target.insertDefault(); + return; + } + + /// extendVariantColumn mutates the variant column in place, so this reference survives + /// addNewVariant below. + auto & variant_column = target.getVariantColumn(); + + if (target.getVariantInfo().variant_name_to_discriminator.contains(type_name) + || target.addNewVariant(getDataTypesCache().getType(type_name), type_name)) + { + const ColumnVariant::Discriminator discriminator + = target.getVariantInfo().variant_name_to_discriminator.at(type_name); + auto & variant = variant_column.getVariantByGlobalDiscriminator(discriminator); + decodeValueIntoColumn(data, pos, context, depth, variant); + variant_column.getOffsets().push_back(variant.size() - 1); + variant_column.getLocalDiscriminators().push_back(variant_column.localDiscriminatorByGlobal(discriminator)); + return; + } + + /// The Dynamic is out of variant slots, so the value goes into the shared variant, which needs + /// it as a standalone column. + const DataTypePtr type = getDataTypesCache().getType(type_name); + auto single_value_column = type->createColumn(); + decodeValueIntoColumn(data, pos, context, depth, *single_value_column); + target.insertValueIntoSharedVariant(*single_value_column, type, type_name, 0); +} + +const ColumnString & unwrapLeaf(const IColumn & column, const NullMap *& out_null_map) +{ + const IColumn * inner = &column; + if (const auto * nullable = typeid_cast(&column)) + { + out_null_map = &nullable->getNullMapData(); + inner = &nullable->getNestedColumn(); + } + return assert_cast(*inner); +} + +} + +void decodeVariantColumn( + const IColumn & metadata, + const IColumn & value, + IColumn & output, + const DataTypePtr & output_type, + const String & column_name, + size_t num_rows, + const FormatSettings & format_settings) +{ + const NullMap * metadata_nulls = nullptr; + const ColumnString & metadata_strings = unwrapLeaf(metadata, metadata_nulls); + + const NullMap * value_nulls = nullptr; + const ColumnString & value_strings = unwrapLeaf(value, value_nulls); + + /// `Nullable(JSON)`: values are decoded into the nested column, nulls (of the whole group or a variant + /// null) go through `output.insertDefault()`, which inserts NULL into a nullable column. + IColumn * output_inner = &output; + NullMap * output_null_map = nullptr; + DataTypePtr inner_type = output_type; + if (auto * output_nullable = typeid_cast(&output)) + { + output_inner = &output_nullable->getNestedColumn(); + output_null_map = &output_nullable->getNullMapData(); + inner_type = removeNullable(output_type); + } + + auto * output_dynamic = typeid_cast(output_inner); + auto * output_object = typeid_cast(output_inner); + std::optional filter; + if (output_object) + filter.emplace( + assert_cast(*inner_type), format_settings.json.type_json_use_partial_match_to_skip_paths_by_regexp); + else + chassert(output_dynamic); + + for (size_t row = 0; row < num_rows; ++row) + { + const std::string_view metadata_blob = metadata_strings.getDataAt(row); + const std::string_view value_blob = value_strings.getDataAt(row); + + /// `metadata` and `value` are one variant value, so they are null together, when the whole + /// variant is null. With `input_format_null_as_default` the leaves are read as non-nullable + /// and a null arrives as an empty blob, which is never a valid encoding (both start with a + /// header byte). A null in just one of the leaves is malformed: in the unshredded layout, + /// which is the only one read here, both fields are required. + const bool metadata_missing = (metadata_nulls && (*metadata_nulls)[row]) || metadata_blob.empty(); + const bool value_missing = (value_nulls && (*value_nulls)[row]) || value_blob.empty(); + if (metadata_missing && value_missing) + { + output.insertDefault(); + continue; + } + if (metadata_missing || value_missing) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed Parquet variant column '{}': a row has {} but no {}", + column_name, metadata_missing ? "`value`" : "`metadata`", metadata_missing ? "`metadata`" : "`value`"); + + const Metadata parsed_metadata = parseMetadata(metadata_blob); + const DecodeContext context{.metadata = parsed_metadata, .max_depth = format_settings.max_parser_depth}; + + if (output_dynamic) + { + decodeValueIntoDynamic(value_blob, 0, context, 0, *output_dynamic); + if (output_null_map) + output_null_map->push_back(false); + continue; + } + + const UInt8 header = readByte(value_blob, 0); + if (BasicType(header & 0x03) != BasicType::Object) + { + const String type_name = getValueTypeName(value_blob, 0); + /// A variant null is an empty object, the same as a null of the whole column. + if (type_name.empty()) + { + output.insertDefault(); + continue; + } + throw Exception( + ErrorCodes::TYPE_MISMATCH, + "Cannot read Parquet variant column '{}' as {}: it contains a value of type {}, but only objects can be " + "read as JSON. Read the column as Dynamic instead", + column_name, output_type->getName(), type_name); + } + + checkDepth(context, 0); + decodeObjectIntoJSON(value_blob, 1, header >> 2, context, 0, *output_object, &*filter); + if (output_null_map) + output_null_map->push_back(false); + } +} + +} diff --git a/src/Processors/Formats/Impl/Parquet/VariantEncoding.h b/src/Processors/Formats/Impl/Parquet/VariantEncoding.h new file mode 100644 index 0000000000000..5e78d00f9228e --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/VariantEncoding.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +namespace DB +{ +struct FormatSettings; +} + +namespace DB::Parquet +{ + +/// `output` must be a column of `output_type`, which is either `Dynamic` or `JSON`. +void decodeVariantColumn( + const IColumn & metadata, + const IColumn & value, + IColumn & output, + const DataTypePtr & output_type, + const String & column_name, + size_t num_rows, + const FormatSettings & format_settings); + +} diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 452a0a102c85d..f96d8b65ff5ce 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -501,6 +501,7 @@ The table below shows how Parquet data types match ClickHouse [data types](/refe | `MultiLineString` (GeoParquet) | [MultiLineString](/reference/data-types/geo#multilinestring) | | `MultiPolygon` (GeoParquet) | [MultiPolygon](/reference/data-types/geo#multipolygon) | | mixed/unknown geometry (GeoParquet) | [Geometry](/reference/data-types/geo#geometry) | +| `VARIANT` | [Dynamic](/reference/data-types/dynamic) | When writing Parquet file, data types that don't have a matching Parquet type are converted to the nearest available type: @@ -544,6 +545,16 @@ For some Parquet types there's no closely matching ClickHouse type. We read them * `TIMESTAMP`/`TIME` with `isAdjustedToUTC=false` is a local wall-clock time (year, month, day, hour, minute, second and subsecond fields in a local timezone, regardless of what specific time zone is considered local), same as SQL `TIMESTAMP WITHOUT TIME ZONE`. ClickHouse reads it as if it were a UTC timestamp instead. E.g. `2025-09-29 18:42:13.000` (representing a reading of a local wall clock) becomes `2025-09-29 18:42:13.000` (`DateTime64(3, 'UTC')` representing a point in time). If converted to String, it shows the correct year, month, day, hour, minute, second and subsecond, which can then be interpreted as being in some local timezone instead of UTC. Counterintuitively, changing the type from `DateTime64(3, 'UTC')` to `DateTime64(3)` would not help as both types represent a point in time rather than a clock reading, but `DateTime64(3)` would incorrectly be formatted using local timezone. * `INTERVAL` is currently read as `FixedString(12)` with raw binary representation of the time interval, as encoded in Parquet file. +## Variant type {#variant-type} + +ClickHouse reads columns stored in the Parquet [`VARIANT`](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) encoding, such as those written by Spark 4.0, into the [`Dynamic`](/reference/data-types/dynamic) type. A column is recognized as a variant if it is a group annotated with the `VARIANT` logical type (specification version 1). Spark 4.0 writes variant columns without this annotation, so a group with just a `metadata` and a `value` field of type `BYTE_ARRAY` is also read as a variant. This can be disabled with the setting `input_format_parquet_detect_variant_by_structure`, then such a group is read as a `Tuple` of the two raw fields, as before; requesting the `Dynamic` or `JSON` type for the column explicitly reads it as a variant regardless of the setting. + +Each value keeps its own type. Variant primitive types are read as the matching ClickHouse types (for example, `int32` as `Int32`, `decimal8` as `Decimal(18, S)`, `timestamp` as `DateTime64(6, 'UTC')`, `timestamp_ntz` as `DateTime64(6)`, `binary` and `string` as `String`, `uuid` as `UUID`), the variant `null` is read as `NULL`, variant objects are read as [`JSON`](/reference/data-types/newjson) with nested objects flattened into dot-separated paths (a `null` field is treated as absent), variant arrays whose elements are all objects are read as `Array(JSON)`, and other variant arrays are read as `Array(Dynamic)`. The top-level type is `Dynamic` rather than `JSON`, because a variant value is not necessarily an object. + +A variant column can also be read as [`JSON`](/reference/data-types/newjson) by requesting that type explicitly, for example in the table structure or in the structure argument of the `file` table function. Then every value must be an object, otherwise an exception is thrown. A variant `null` and a `null` of the whole column are read as an empty object. Typed paths of the requested `JSON` type are converted to their types, and paths listed in `SKIP` and `SKIP REGEXP` are skipped. + +Shredded variants (groups with a `typed_value` field) are not supported yet: reading such a column throws an exception, while the other columns of the file can still be read, and schema inference skips the column if `input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference` is enabled. Writing `Dynamic` or `JSON` columns to Parquet as `VARIANT` is not supported either. + ## Geo types (GeoParquet) {#geo-types} ClickHouse supports reading and writing geometry columns according to the [GeoParquet](https://geoparquet.org/) specification. Geometry columns are stored as `BYTE_ARRAY` payloads encoded in [WKB](https://libgeos.org/specifications/wkb/) (or WKT on read), with a JSON `geo` key in the file-level Parquet metadata describing each geometry column's encoding, geometry type and CRS. diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 459655fda1ddc..0f0d63690f2a6 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -65,6 +65,25 @@ EXCLUDE=( # malformed on purpose, so loading it here would print an exception. 05035_delta_byte_array_zero_values.parquet 05035_delta_byte_array_decimal.parquet + # Variant fixtures for the 04928-04933 tests. This test hashes a file through GROUP BY, which + # rejects the Dynamic columns they are read as, so there is nothing for it to check here. + 04928_variant_spark.parquet + 04929_variant_all_types.parquet + 04930_variant_shredded.parquet + 04931_variant_max_dynamic_types.parquet + 04932_variant_null.parquet + 04933_variant_binary_and_string.parquet + 05227_variant_nested_types.parquet + 05296_variant_json_objects.parquet + 05297_variant_json_shared_data.parquet + # Its object has a duplicate path on purpose, so loading it here would print an exception. + 05298_variant_json_duplicate_path.parquet + 05299_variant_read_as_json.parquet + # Intentionally truncated variant blobs for the 05228 malformed-input test. + 05228_variant_malformed.parquet + # Unsupported `VARIANT` specification version and one-sided null leaves, on purpose. + 05306_variant_unsupported_version.parquet + 05307_variant_one_sided_null.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/04928_parquet_variant_read.reference b/tests/queries/0_stateless/04928_parquet_variant_read.reference new file mode 100644 index 0000000000000..8556224ae976a --- /dev/null +++ b/tests/queries/0_stateless/04928_parquet_variant_read.reference @@ -0,0 +1,13 @@ +1 100000 +2 hello +--- schema inference --- +n Int32 +v Dynamic +--- input_format_parquet_detect_variant_by_structure = 0 --- +n Int32 +v Tuple(value String, metadata String) +1 5 3 +2 6 3 +--- explicit Dynamic bypasses the setting --- +1 100000 +2 hello diff --git a/tests/queries/0_stateless/04928_parquet_variant_read.sh b/tests/queries/0_stateless/04928_parquet_variant_read.sh new file mode 100755 index 0000000000000..a7b9655f8ac56 --- /dev/null +++ b/tests/queries/0_stateless/04928_parquet_variant_read.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Written by Spark 4.0.0: `SELECT cast(100000 as variant), cast('hello' as variant)`. An unshredded +# variant column is a group with a `value` and a `metadata` BYTE_ARRAY leaf, in that order, with no +# `VARIANT` logical type - Spark records the variant-ness only in its own key-value metadata +# `org.apache.spark.sql.parquet.row.metadata`. So the group has to be recognized by structure. +# +# required group spark_schema { +# required int32 n; +# required group v { +# required binary value; +# required binary metadata; +# } +# } +DATA_FILE=$CUR_DIR/data_parquet/04928_variant_spark.parquet + +${CLICKHOUSE_LOCAL} --stacktrace --query="SELECT n, v FROM file('${DATA_FILE}', Parquet) ORDER BY n" + +# Without the `VARIANT` logical type the group is recognized by structure, which is what the setting +# controls. Before variant support such a group was read as a Tuple of the raw blobs. +echo '--- schema inference ---' +${CLICKHOUSE_LOCAL} --query="DESCRIBE file('${DATA_FILE}', Parquet)" +echo '--- input_format_parquet_detect_variant_by_structure = 0 ---' +${CLICKHOUSE_LOCAL} --query="DESCRIBE file('${DATA_FILE}', Parquet) SETTINGS input_format_parquet_detect_variant_by_structure = 0, print_pretty_type_names = 0" +${CLICKHOUSE_LOCAL} --query=" + SELECT n, length(v.value), length(v.metadata) FROM file('${DATA_FILE}', Parquet) ORDER BY n + SETTINGS input_format_parquet_detect_variant_by_structure = 0" +echo '--- explicit Dynamic bypasses the setting ---' +${CLICKHOUSE_LOCAL} --query=" + SELECT n, v FROM file('${DATA_FILE}', Parquet, 'n Int32, v Dynamic') ORDER BY n + SETTINGS input_format_parquet_detect_variant_by_structure = 0" diff --git a/tests/queries/0_stateless/04929_parquet_variant_all_types.reference b/tests/queries/0_stateless/04929_parquet_variant_all_types.reference new file mode 100644 index 0000000000000..56d410f457234 --- /dev/null +++ b/tests/queries/0_stateless/04929_parquet_variant_all_types.reference @@ -0,0 +1,27 @@ +1 None \N +2 Bool true +3 Bool false +4 Int8 -128 +5 Int16 -32768 +6 Int32 100000 +7 Int64 1234567890123 +8 Float64 3.5 +9 Decimal(9, 2) 123.45 +10 Decimal(18, 4) 12345678.9012 +11 Decimal(38, 10) 12345678901234567890.123456789 +12 Date32 2026-08-28 +13 DateTime64(6, \'UTC\') 2026-08-28 12:34:56.789012 +14 DateTime64(6) 2026-08-28 12:34:56.789012 +15 Float32 1.5 +16 String bin +17 String yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy +18 Time64(6) 12:34:56.789012 +19 DateTime64(9, \'UTC\') 2026-08-28 12:34:56.789012000 +20 DateTime64(9) 2026-08-28 12:34:56.789012000 +21 UUID 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +22 String hello +23 JSON {"a":1,"b":"x"} +24 Array(Dynamic) [1,2,'three'] +25 JSON {"arr":[1,2],"obj":{"k":7}} +26 JSON {} +27 Array(Dynamic) [] diff --git a/tests/queries/0_stateless/04929_parquet_variant_all_types.sh b/tests/queries/0_stateless/04929_parquet_variant_all_types.sh new file mode 100755 index 0000000000000..d63949d9fa064 --- /dev/null +++ b/tests/queries/0_stateless/04929_parquet_variant_all_types.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# One unshredded variant column covering every primitive type id (0..20) of the Apache variant +# encoding and all four basic types: primitive, short string, object, array. +DATA_FILE=$CUR_DIR/data_parquet/04929_variant_all_types.parquet + +${CLICKHOUSE_LOCAL} --query=" + SELECT n, dynamicType(v) AS type, v + FROM file('${DATA_FILE}', Parquet) + ORDER BY n + SETTINGS session_timezone = 'UTC' +" diff --git a/tests/queries/0_stateless/04930_parquet_variant_shredded.reference b/tests/queries/0_stateless/04930_parquet_variant_shredded.reference new file mode 100644 index 0000000000000..e99b1d9ed13cb --- /dev/null +++ b/tests/queries/0_stateless/04930_parquet_variant_shredded.reference @@ -0,0 +1,8 @@ +--- reading the column --- +Parquet column v is a shredded variant (it has a `typed_value` field), which is not supported yet +--- reading the other column --- +1 +2 +--- schema inference --- +Parquet column v is a shredded variant (it has a `typed_value` field), which is not supported yet +n Int32 diff --git a/tests/queries/0_stateless/04930_parquet_variant_shredded.sh b/tests/queries/0_stateless/04930_parquet_variant_shredded.sh new file mode 100755 index 0000000000000..89baec2563fb7 --- /dev/null +++ b/tests/queries/0_stateless/04930_parquet_variant_shredded.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A shredded variant column: the group has a `typed_value` leaf next to `metadata` and `value`. +# Row 1 fits the shredded type and lives in `typed_value`; row 2 does not, so it falls back to the +# variant-encoded `value`. Shredding is not supported yet, so reading the column is rejected +# instead of silently losing row 1. The other columns of the file stay readable, and schema +# inference can skip the column. +# +# required group v { +# required binary metadata; +# optional binary value; +# optional int32 typed_value; +# } +DATA_FILE=$CUR_DIR/data_parquet/04930_variant_shredded.parquet + +echo '--- reading the column ---' +${CLICKHOUSE_LOCAL} --query="SELECT n, v FROM file('${DATA_FILE}', Parquet) ORDER BY n" 2>&1 \ + | grep -o "Parquet column v is a shredded variant.*not supported yet" + +echo '--- reading the other column ---' +${CLICKHOUSE_LOCAL} --query="SELECT n FROM file('${DATA_FILE}', Parquet, 'n Int32') ORDER BY n" + +echo '--- schema inference ---' +${CLICKHOUSE_LOCAL} --query="DESCRIBE file('${DATA_FILE}', Parquet)" 2>&1 \ + | grep -o "Parquet column v is a shredded variant.*not supported yet" +${CLICKHOUSE_LOCAL} --query=" + DESCRIBE file('${DATA_FILE}', Parquet) + SETTINGS input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference = 1" diff --git a/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.reference b/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.reference new file mode 100644 index 0000000000000..1c65de72b3a80 --- /dev/null +++ b/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.reference @@ -0,0 +1,73 @@ +-- every row keeps its own decimal type +Decimal(9, 0) 12345 +Decimal(9, 1) 1234.5 +Decimal(9, 2) 123.45 +Decimal(9, 3) 12.345 +Decimal(9, 4) 1.2345 +Decimal(9, 5) 0.12345 +Decimal(9, 6) 0.012345 +Decimal(9, 7) 0.0012345 +Decimal(9, 8) 0.00012345 +Decimal(9, 9) 0.000012345 +Decimal(18, 0) 12345 +Decimal(18, 1) 1234.5 +Decimal(18, 2) 123.45 +Decimal(18, 3) 12.345 +Decimal(18, 4) 1.2345 +Decimal(18, 5) 0.12345 +Decimal(18, 6) 0.012345 +Decimal(18, 7) 0.0012345 +Decimal(18, 8) 0.00012345 +Decimal(18, 9) 0.000012345 +Decimal(18, 10) 0.0000012345 +Decimal(18, 11) 0.00000012345 +Decimal(18, 12) 0.000000012345 +Decimal(18, 13) 0.0000000012345 +Decimal(18, 14) 0.00000000012345 +Decimal(18, 15) 0.000000000012345 +Decimal(18, 16) 0.0000000000012345 +Decimal(18, 17) 0.00000000000012345 +Decimal(18, 18) 0.000000000000012345 +Decimal(38, 0) 12345 +Decimal(38, 1) 1234.5 +Decimal(38, 2) 123.45 +Decimal(38, 3) 12.345 +Decimal(38, 4) 1.2345 +Decimal(38, 5) 0.12345 +Decimal(38, 6) 0.012345 +Decimal(38, 7) 0.0012345 +Decimal(38, 8) 0.00012345 +Decimal(38, 9) 0.000012345 +Decimal(38, 10) 0.0000012345 +Decimal(38, 11) 0.00000012345 +Decimal(38, 12) 0.000000012345 +Decimal(38, 13) 0.0000000012345 +Decimal(38, 14) 0.00000000012345 +Decimal(38, 15) 0.000000000012345 +Decimal(38, 16) 0.0000000000012345 +Decimal(38, 17) 0.00000000000012345 +Decimal(38, 18) 0.000000000000012345 +Decimal(38, 19) 0.0000000000000012345 +Decimal(38, 20) 0.00000000000000012345 +Decimal(38, 21) 0.000000000000000012345 +Decimal(38, 22) 0.0000000000000000012345 +Decimal(38, 23) 0.00000000000000000012345 +Decimal(38, 24) 0.000000000000000000012345 +Decimal(38, 25) 0.0000000000000000000012345 +Decimal(38, 26) 0.00000000000000000000012345 +Decimal(38, 27) 0.000000000000000000000012345 +Decimal(38, 28) 0.0000000000000000000000012345 +Decimal(38, 29) 0.00000000000000000000000012345 +Decimal(38, 30) 0.000000000000000000000000012345 +Decimal(38, 31) 0.0000000000000000000000000012345 +Decimal(38, 32) 0.00000000000000000000000000012345 +Decimal(38, 33) 0.000000000000000000000000000012345 +Decimal(38, 34) 0.0000000000000000000000000000012345 +Decimal(38, 35) 0.00000000000000000000000000000012345 +Decimal(38, 36) 0.000000000000000000000000000000012345 +Decimal(38, 37) 0.0000000000000000000000000000000012345 +Decimal(38, 38) 0.00000000000000000000000000000000012345 +-- distinct types read back +68 39 +-- types beyond the limit live in the shared variant +1 diff --git a/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.sh b/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.sh new file mode 100755 index 0000000000000..d1a3597c1dafa --- /dev/null +++ b/tests/queries/0_stateless/04931_parquet_variant_max_dynamic_types.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An unshredded variant column carrying more distinct types than `max_dynamic_types` (32 by +# default): 68 decimals, one per valid (precision, scale) pair of the three variant decimal type +# ids. Every scale is a distinct ClickHouse type, so the types past the limit have to go into the +# Dynamic shared variant. The unscaled value is 12345 in every row, so a scale dropped on the way +# into the column shows up as 12345 instead of 0.00012345 and friends. +DATA_FILE=$CUR_DIR/data_parquet/04931_variant_max_dynamic_types.parquet + +echo "-- every row keeps its own decimal type" +${CLICKHOUSE_LOCAL} --query=" + SELECT dynamicType(v) AS type, toString(v) AS value + FROM file('${DATA_FILE}', Parquet)" + +echo "-- distinct types read back" +${CLICKHOUSE_LOCAL} --query=" + SELECT uniqExact(dynamicType(v)), uniqExact(toString(v)) + FROM file('${DATA_FILE}', Parquet)" + +echo "-- types beyond the limit live in the shared variant" +${CLICKHOUSE_LOCAL} --query=" + SELECT countIf(isDynamicElementInSharedData(v)) > 0 + FROM file('${DATA_FILE}', Parquet)" diff --git a/tests/queries/0_stateless/04932_parquet_variant_null_as_default.reference b/tests/queries/0_stateless/04932_parquet_variant_null_as_default.reference new file mode 100644 index 0000000000000..dd1cc4e4295e8 --- /dev/null +++ b/tests/queries/0_stateless/04932_parquet_variant_null_as_default.reference @@ -0,0 +1,8 @@ +-- input_format_null_as_default = 0 +Int8 7 +None \N +Int8 9 +-- input_format_null_as_default = 1 +Int8 7 +None \N +Int8 9 diff --git a/tests/queries/0_stateless/04932_parquet_variant_null_as_default.sh b/tests/queries/0_stateless/04932_parquet_variant_null_as_default.sh new file mode 100755 index 0000000000000..8909de77adcff --- /dev/null +++ b/tests/queries/0_stateless/04932_parquet_variant_null_as_default.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An unshredded variant column whose middle row is null. With `input_format_null_as_default` the +# leaves are read as non-nullable and the null arrives as an empty blob instead, which is not a +# well-formed variant value (both leaves start with a header byte). It has to read as a null +# Dynamic, the same as through the null map. +DATA_FILE=$CUR_DIR/data_parquet/04932_variant_null.parquet + +for null_as_default in 0 1; do + echo "-- input_format_null_as_default = ${null_as_default}" + ${CLICKHOUSE_LOCAL} --query=" + SELECT dynamicType(v) AS type, v + FROM file('${DATA_FILE}', Parquet) + SETTINGS input_format_null_as_default = ${null_as_default}" +done diff --git a/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.reference b/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.reference new file mode 100644 index 0000000000000..fda788a3e55d0 --- /dev/null +++ b/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.reference @@ -0,0 +1,6 @@ +String 5 68656C6C6F +String 4 00FF01FE +String 12 D0BFD180D0B8D0B2D0B5D182 +String 0 +-- one variant for all three encodings +1 diff --git a/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.sh b/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.sh new file mode 100755 index 0000000000000..5fe0b62537595 --- /dev/null +++ b/tests/queries/0_stateless/04933_parquet_variant_binary_and_string.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An unshredded variant column with the three payload-carrying encodings of the variant spec: the +# `string` primitive, the `binary` primitive and a short string. ClickHouse has no type distinct +# from String for binary, so all three land in one String variant rather than splitting the column +# into two types; the bytes must survive verbatim, including a payload that is not valid UTF-8 and +# contains an embedded zero byte. +DATA_FILE=$CUR_DIR/data_parquet/04933_variant_binary_and_string.parquet + +${CLICKHOUSE_LOCAL} --query=" + SELECT dynamicType(v) AS type, length(v::String) AS bytes, hex(v::String) AS payload + FROM file('${DATA_FILE}', Parquet)" + +echo "-- one variant for all three encodings" +${CLICKHOUSE_LOCAL} --query=" + SELECT uniqExact(dynamicType(v)) + FROM file('${DATA_FILE}', Parquet)" diff --git a/tests/queries/0_stateless/05227_parquet_variant_nested_types.reference b/tests/queries/0_stateless/05227_parquet_variant_nested_types.reference new file mode 100644 index 0000000000000..7039964f1b2bb --- /dev/null +++ b/tests/queries/0_stateless/05227_parquet_variant_nested_types.reference @@ -0,0 +1,11 @@ +1 {'a':'Int8','b':'String'} +2 {'d':'Date32','t':'Bool'} +3 {'dec':'Decimal(9, 2)','ts':'DateTime64(6, \'UTC\')'} +4 {'dt':'DateTime64(6)','u':'UUID'} +5 {'a':'String','b':'Int8'} +6 {'a':'String','b.c':'Date32'} +--- values --- +2 {"d":"2026-08-28","t":true} +3 {"dec":123.45,"ts":"2025-09-04 12:34:56.789012"} +4 {"dt":"2025-09-04 12:34:56.789012","u":"00010203-0405-0607-0809-0a0b0c0d0e0f"} +6 {"a":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz","b":{"c":"2026-08-28"}} diff --git a/tests/queries/0_stateless/05227_parquet_variant_nested_types.sh b/tests/queries/0_stateless/05227_parquet_variant_nested_types.sh new file mode 100755 index 0000000000000..0d23e0107542d --- /dev/null +++ b/tests/queries/0_stateless/05227_parquet_variant_nested_types.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Values nested inside a variant object keep their own type, nested objects are flattened into +# `JSON` paths, and object/array offsets wider than one signed byte are read as unsigned. +DATA_FILE=$CUR_DIR/data_parquet/05227_variant_nested_types.parquet + +${CLICKHOUSE_LOCAL} --query " + SELECT n, JSONAllPathsWithTypes(dynamicElement(v, 'JSON')) AS nested_types + FROM file('${DATA_FILE}', Parquet) + ORDER BY n + SETTINGS session_timezone = 'UTC' +" + +echo '--- values ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, v + FROM file('${DATA_FILE}', Parquet) + WHERE n IN (2, 3, 4, 6) + ORDER BY n + SETTINGS session_timezone = 'UTC' +" diff --git a/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.reference b/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.reference new file mode 100644 index 0000000000000..87924c88b8656 --- /dev/null +++ b/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.reference @@ -0,0 +1,5 @@ +Malformed Parquet variant: 16 bytes are needed at offset 1, but the blob is 1 bytes +Malformed Parquet variant: 1 bytes are needed at offset 1, but the blob is 1 bytes +Malformed Parquet variant: 8 bytes are needed at offset 2, but the blob is 2 bytes +Malformed Parquet variant: 99 bytes are needed at offset 5, but the blob is 5 bytes +Malformed Parquet variant: 3 bytes are needed at offset 1, but the blob is 1 bytes diff --git a/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.sh b/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.sh new file mode 100755 index 0000000000000..a477e6486d2a9 --- /dev/null +++ b/tests/queries/0_stateless/05228_parquet_variant_malformed_blob.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every row has a value header promising more payload than the blob holds. Each must be reported as +# malformed data rather than read out of bounds. +DATA_FILE=$CUR_DIR/data_parquet/05228_variant_malformed.parquet + +for n in 1 2 3 4 5; do + ${CLICKHOUSE_LOCAL} --query "SELECT v FROM file('${DATA_FILE}', Parquet) WHERE n = $n" 2>&1 \ + | grep -o "Malformed Parquet variant: [0-9]* bytes are needed at offset [0-9]*, but the blob is [0-9]* bytes" +done diff --git a/tests/queries/0_stateless/05296_parquet_variant_json_objects.reference b/tests/queries/0_stateless/05296_parquet_variant_json_objects.reference new file mode 100644 index 0000000000000..e2ad63f78a180 --- /dev/null +++ b/tests/queries/0_stateless/05296_parquet_variant_json_objects.reference @@ -0,0 +1,13 @@ +1 JSON {"a":1,"b":{"c":"x","d":{"e":true}}} +2 JSON {"f":[1,"s"]} +3 Array(JSON) ['{"k":1}','{"k":2,"m":"z"}'] +4 Array(Dynamic) ['{"k":1}',2] +5 JSON {"arr":[{"x":1},{"y":{"z":2}}]} +6 JSON {} +7 Int64 42 +8 Array(Dynamic) [] +--- paths --- +1 {'a':'Int64','b.c':'String','b.d.e':'Bool'} +2 {'f':'Array(Dynamic)'} +5 {'arr':'Array(JSON)'} +6 {} diff --git a/tests/queries/0_stateless/05296_parquet_variant_json_objects.sh b/tests/queries/0_stateless/05296_parquet_variant_json_objects.sh new file mode 100755 index 0000000000000..35a2c9bc36c8a --- /dev/null +++ b/tests/queries/0_stateless/05296_parquet_variant_json_objects.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Variant objects are read as `JSON` with nested objects flattened into paths and null fields +# dropped. Arrays of objects are read as `Array(JSON)`, other arrays as `Array(Dynamic)`. +DATA_FILE=$CUR_DIR/data_parquet/05296_variant_json_objects.parquet + +${CLICKHOUSE_LOCAL} --query " + SELECT n, dynamicType(v) AS type, v + FROM file('${DATA_FILE}', Parquet) + ORDER BY n +" + +echo '--- paths ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, JSONAllPathsWithTypes(dynamicElement(v, 'JSON')) AS paths + FROM file('${DATA_FILE}', Parquet) + WHERE dynamicType(v) = 'JSON' + ORDER BY n +" diff --git a/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.reference b/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.reference new file mode 100644 index 0000000000000..f0ce01b4dffdd --- /dev/null +++ b/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.reference @@ -0,0 +1 @@ +1024 76 0 1023 1099 Int64 diff --git a/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.sh b/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.sh new file mode 100755 index 0000000000000..1f784e0573c9e --- /dev/null +++ b/tests/queries/0_stateless/05297_parquet_variant_json_shared_data.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A variant object with 1100 fields: paths over the `max_dynamic_paths` limit (1024) of `JSON` +# go to the shared data. +DATA_FILE=$CUR_DIR/data_parquet/05297_variant_json_shared_data.parquet + +${CLICKHOUSE_LOCAL} --query " + SELECT + length(JSONDynamicPaths(j)), + length(JSONSharedDataPaths(j)), + j.k0000, + j.k1023, + j.k1099, + dynamicType(j.k1099) + FROM (SELECT dynamicElement(v, 'JSON') AS j FROM file('${DATA_FILE}', Parquet)) +" diff --git a/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.reference b/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.reference new file mode 100644 index 0000000000000..d08d3c723e2ea --- /dev/null +++ b/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.reference @@ -0,0 +1 @@ +INCORRECT_DATA diff --git a/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.sh b/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.sh new file mode 100755 index 0000000000000..ee020f147037c --- /dev/null +++ b/tests/queries/0_stateless/05298_parquet_variant_json_duplicate_path.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The variant object {"a.b": 1, "a": {"b": 2}} has the path `a.b` twice once nested objects are +# flattened into `JSON` paths. +DATA_FILE=$CUR_DIR/data_parquet/05298_variant_json_duplicate_path.parquet + +${CLICKHOUSE_LOCAL} --query "SELECT v FROM file('${DATA_FILE}', Parquet)" 2>&1 | grep -o -m1 'INCORRECT_DATA' diff --git a/tests/queries/0_stateless/05299_parquet_variant_read_as_json.reference b/tests/queries/0_stateless/05299_parquet_variant_read_as_json.reference new file mode 100644 index 0000000000000..60fc26c783837 --- /dev/null +++ b/tests/queries/0_stateless/05299_parquet_variant_read_as_json.reference @@ -0,0 +1,52 @@ +1 {"a":1,"b":{"c":"x"},"s":"hello"} {'a':'Int8','b.c':'String','s':'String'} +2 {"a":300,"tags":["x","y"]} {'a':'Int16','tags':'Array(Dynamic)'} +3 {} {} +4 {} {} +5 {} {} +6 {"a":5,"b":{"c":"y","d":1}} {'a':'Int8','b.c':'String','b.d':'Int8'} +--- typed path --- +1 1 Int64 x LowCardinality(String) +2 300 Int64 LowCardinality(String) +3 0 Int64 LowCardinality(String) +4 0 Int64 LowCardinality(String) +5 0 Int64 LowCardinality(String) +6 5 Int64 y LowCardinality(String) +--- skip --- +1 {"a":1,"s":"hello"} +2 {"a":300} +3 {} +4 {} +5 {} +6 {"a":5} +--- shared data --- +1 {"a":1,"b":{"c":"x"},"s":"hello"} ['a'] ['b.c','s'] +2 {"a":300,"tags":["x","y"]} ['a'] ['tags'] +3 {} [] [] +4 {} [] [] +5 {} [] [] +6 {"a":5,"b":{"c":"y","d":1}} ['a'] ['b.c','b.d'] +--- insert into a table --- +1 1 x +2 300 \N +3 0 \N +4 0 \N +5 0 \N +6 5 y +--- Nullable(JSON): the variant null and the null group are NULL --- +1 {"a":1,"b":{"c":"x"},"s":"hello"} Nullable(JSON(a Int64)) 1 x +2 {"a":300,"tags":["x","y"]} Nullable(JSON(a Int64)) 300 \N +3 \N Nullable(JSON(a Int64)) \N \N +4 \N Nullable(JSON(a Int64)) \N \N +5 {"a":0} Nullable(JSON(a Int64)) 0 \N +6 {"a":5,"b":{"c":"y","d":1}} Nullable(JSON(a Int64)) 5 y +1 {"a":1,"b":{"c":"x"},"s":"hello"} 0 +2 {"a":300,"tags":["x","y"]} 0 +3 \N 1 +4 \N 1 +5 {"a":0} 0 +6 {"a":5,"b":{"c":"y","d":1}} 0 +--- schema inference still gives Dynamic --- +n Nullable(Int64) +v Dynamic +--- a non-object value --- +Cannot read Parquet variant column 'v' as JSON: it contains a value of type Int32, but only objects can be read as JSON. Read the column as Dynamic instead diff --git a/tests/queries/0_stateless/05299_parquet_variant_read_as_json.sh b/tests/queries/0_stateless/05299_parquet_variant_read_as_json.sh new file mode 100755 index 0000000000000..8e8c827eaa26f --- /dev/null +++ b/tests/queries/0_stateless/05299_parquet_variant_read_as_json.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A variant column is read as `JSON` when the requested type is `JSON`. Rows 3 and 4 are a variant +# null and a null of the whole group, both are read as an empty object. +DATA_FILE=$CUR_DIR/data_parquet/05299_variant_read_as_json.parquet + +${CLICKHOUSE_LOCAL} --query " + SELECT n, v, JSONAllPathsWithTypes(v) + FROM file('${DATA_FILE}', Parquet, 'n Int64, v JSON') + ORDER BY n +" + +echo '--- typed path ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, v.a, toTypeName(v.a), v.b.c, toTypeName(v.b.c) + FROM file('${DATA_FILE}', Parquet, 'n Int64, v JSON(a Int64, b.c LowCardinality(String))') + ORDER BY n +" + +echo '--- skip ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, v + FROM file('${DATA_FILE}', Parquet, 'n Int64, v JSON(SKIP b, SKIP REGEXP \'^t\')') + ORDER BY n +" + +echo '--- shared data ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, v, JSONDynamicPaths(v), JSONSharedDataPaths(v) + FROM file('${DATA_FILE}', Parquet, 'n Int64, v JSON(max_dynamic_paths = 1)') + ORDER BY n +" + +echo '--- insert into a table ---' +${CLICKHOUSE_LOCAL} --query " + CREATE TABLE t (n Int64, v JSON(a Int64)) ENGINE = Memory; + INSERT INTO t SELECT * FROM file('${DATA_FILE}', Parquet); + SELECT n, v.a, v.b.c FROM t ORDER BY n; +" + +echo '--- Nullable(JSON): the variant null and the null group are NULL ---' +${CLICKHOUSE_LOCAL} --query " + SELECT n, v, toTypeName(v), v.a, v.b.c + FROM file('${DATA_FILE}', Parquet, 'n Int64, v Nullable(JSON(a Int64))') + ORDER BY n +" +${CLICKHOUSE_LOCAL} --query " + CREATE TABLE tn (n Int64, v Nullable(JSON(a Int64))) ENGINE = Memory; + INSERT INTO tn SELECT * FROM file('${DATA_FILE}', Parquet); + SELECT n, v, isNull(v) FROM tn ORDER BY n; +" +echo '--- schema inference still gives Dynamic ---' +${CLICKHOUSE_LOCAL} --query "DESCRIBE file('${DATA_FILE}', Parquet)" + +echo '--- a non-object value ---' +${CLICKHOUSE_LOCAL} --query " + SELECT * FROM file('${CUR_DIR}/data_parquet/04928_variant_spark.parquet', Parquet, 'id Int64, v JSON') +" 2>&1 | grep -o 'Cannot read Parquet variant column .v. as JSON: it contains a value of type Int32.*Read the column as Dynamic instead' diff --git a/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.reference b/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.reference new file mode 100644 index 0000000000000..fbaf8cd007a9f --- /dev/null +++ b/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.reference @@ -0,0 +1,2 @@ +Parquet column v is a variant with specification version 2, but only version 1 is supported +8 8 diff --git a/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.sh b/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.sh new file mode 100755 index 0000000000000..17a787024676b --- /dev/null +++ b/tests/queries/0_stateless/05306_parquet_variant_unsupported_version.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A copy of 05296_variant_json_objects.parquet whose `VARIANT` logical type has +# `specification_version = 2`. Only version 1 exists, so the blobs may not have today's layout: +# the column is rejected instead of being decoded as version 1. The other columns are still readable. +DATA_FILE=$CUR_DIR/data_parquet/05306_variant_unsupported_version.parquet + +${CLICKHOUSE_LOCAL} --query="SELECT n, v FROM file('${DATA_FILE}', Parquet) ORDER BY n" 2>&1 \ + | grep -o "Parquet column v is a variant with specification version 2, but only version 1 is supported" + +${CLICKHOUSE_LOCAL} --query="SELECT count(), max(n) FROM file('${DATA_FILE}', Parquet, 'n Int32')" diff --git a/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.reference b/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.reference new file mode 100644 index 0000000000000..bada9f1e30e77 --- /dev/null +++ b/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.reference @@ -0,0 +1,4 @@ +1 7 +4 \N +Malformed Parquet variant column 'v': a row has `metadata` but no `value` +Malformed Parquet variant column 'v': a row has `value` but no `metadata` diff --git a/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.sh b/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.sh new file mode 100755 index 0000000000000..bc7fe7ddb4d67 --- /dev/null +++ b/tests/queries/0_stateless/05307_parquet_variant_one_sided_null.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format, which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `metadata` and `value` together are one variant value, so in an unshredded variant they can only +# be null together, when the whole group is null (row 4). A null in just one of them (rows 2 and 3) +# is malformed and must be reported, not read as NULL. +# +# optional group v { +# optional binary metadata; +# optional binary value; +# } +DATA_FILE=$CUR_DIR/data_parquet/05307_variant_one_sided_null.parquet + +${CLICKHOUSE_LOCAL} --query="SELECT n, v FROM file('${DATA_FILE}', Parquet) WHERE n IN (1, 4) ORDER BY n" + +for n in 2 3; do + ${CLICKHOUSE_LOCAL} --query="SELECT v FROM file('${DATA_FILE}', Parquet) WHERE n = $n" 2>&1 \ + | grep -o "Malformed Parquet variant column 'v': a row has [^:]*" +done diff --git a/tests/queries/0_stateless/data_parquet/04928_variant_spark.parquet b/tests/queries/0_stateless/data_parquet/04928_variant_spark.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b5a0899832c0e686d9c0c45135fc3eaa14ae0bc4 GIT binary patch literal 921 zcmaJ=&x_MQ6n^>9B`if0c7{Z9D1i+v)X;XDZd-Ct*n@alq&F3rwv*jrns$>ki@jL(YMiFzkl~~dm~)J(9WwT5^QB>j(HHwF!u`Dmye&{Ye-r?_mekAp(?t|JuCwSK_>8Od8a_%DrATLl1Lvi zyt2TnN@-3kPzQ8iA9x+87E0*3G8$_MgG_F&;Zy_R*0l>npZfNhWWSJ1 zp`RGA(o9RS2BR+SO3v>O%cbQt!&ilv^g<>bpg$z4+8RzfsB&G`XVHhA8tc!crIkFMbE|K~NwgRxR^9R^JQ&AT z|K8FH*)$7gb6?55pINwAYj7)<30;wUbikmt^RVgs;`sNKglVJCM?W2{-69{ z1^S($-W$`XQZ!g;I!)JURjR|qBp6q^ZjZB3v*QjMUBB1b_gS;;_8OzTz2V69+xwk% W!}W)qZq4vafc%4J_zDj3QT_%}HMW`n literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04929_variant_all_types.parquet b/tests/queries/0_stateless/data_parquet/04929_variant_all_types.parquet new file mode 100644 index 0000000000000000000000000000000000000000..855ad60eff63fb17cb020e0ffdd0d54ff9abce33 GIT binary patch literal 1476 zcmaJ>TWAzl82-X^wqSWLTL*^OG}|3J}JIPo+KcM|37QIB|7YPF8`eKegFTR zGh2~;9YPiVG4Y7?ikyHo0F8wa1^^tPAXJ2g&=Cd42oMq7rN)D6m?X}!G`pEvjoxF{aP$_LH@8aedEE^ zbJ^c}Zr;t05RED>>f%pzsfszpA_ia@s7$*MrE-i}D#(Iaswa7Ej)jFdf2c1{e#JmD zRSz`QZ|tQ~Q+#K(RpbUW*>+`~ec4YHrQ?6a_Z)csc=Fzh)z2=G0;?LCe=4ALdfBAz z-fBClYbI<@n5=z69^ltKsBNv^tailONd~W~RDR^S>nW ziw_-ULT|!cC{eMMVBx-c>~`YmPv@?~@ZbMmqs&CT)@Y2A^_&9(t{O}`O|B~nPT~eL zxR|OpYqfP?wi_`^;ad8z!8zl=jR-k_UQmZX^Yfzc&iovt6U#&$2BGy{gG|h^PaZ&7 zlzZ_E>FC=%Qb2zqEaG6AJQy(9s*D1P9toPw@qytH)Ha0mfwsxsU*IUXKUCI}JOj$t zc*yRt9pLDbgZ0{IAXE^1BRFgfFvzB9ZfxX^HXSrh)o2V(bhI`V91o@fuyd~G7JTpo zcXxnpxwuboJ7=F)?2C#&$NlK%@1C2%JBEs6nuQ$zkr^$rG@EY8hLrOW`@HVYM*W`^ z?=zNu%viARsIKn%kXuInwvLehKC^#g{$k9(93%dfZClPh&ybz2ko}!u=lrPsZya3D zF=Sugw}*DMOm!Cd7J5$0qLi|Vpl6?yGqN}&%Tf->(oh&H;)4A&DoNQxJFeV)8NF!Ci+ZfQq#5OTa17QXeumAu6 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04930_variant_shredded.parquet b/tests/queries/0_stateless/data_parquet/04930_variant_shredded.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7fce85375c0c11064bba52a12bc8a9b83673b549 GIT binary patch literal 1238 zcmaJ>OK;Oa5Z+xID-uzaMqO(y`H%}%kZ79(T2aJhokvpV+oZVQEiXfe8VpL8_)?2Ag=1jwI?N_18W0vJbfZb2afPqU8!c$${dh1K z0);%!Ur^X&MN4NysucqAG8kouli`^;&X3O&hq#0wOpI!2v*LnxfkYLOGuR-)8b|&3 zIOIT)-6MjY5D3yKw+PRIpE-^v;ZZmV(PiS!G#s49SjxoR$;VNAbf6=)>9|hIBQ9so z2C;GuAaP=w_E!pz_sGlKzFh}b0Ju0#4iq7XmDN~9Iz8v1KodLh{!#B6b+7FDD$G+m z>sSu1`!bxKV}amr=6zxAjoZH&(_?dz>A*m8aU%-*jM`_^`(t?D4H*BH?{e*q`199>#}!%G>Ac*1qQ9B=Ypmlb^?WI1056_rQqH8`lMN|*vf5KI9}1qMRh4oF zHTZK49-6gxX&lVg0Qwl+7fqDV1gEa#4CD@8=J=l$N3; z$ME{4c+Ven8(WqA-4Eq%;78E3uF|iR45T06)!R}=vW6ibt?vUn2-LigL`O*!&?05C z+dpi+8K7TAK3lbvkF2Zy#d)v@_5_`i?p~+g-ENfjH5`-z1x$m0W$Jy+^d40bb9nkn zMkrOiay9%#DcoY5K$O~$@|lIvKoy>t6`E1XN75;jNcAV3YCXzrM0rt QatS$q$S?TmwBe8TA0sB>q5uE@ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04931_variant_max_dynamic_types.parquet b/tests/queries/0_stateless/data_parquet/04931_variant_max_dynamic_types.parquet new file mode 100644 index 0000000000000000000000000000000000000000..04292c77a8ade7ef56f670cc5ac9b996bce16d91 GIT binary patch literal 1217 zcmajf%}*0S6aeryTe_5jwJpxNixn$E(-1(wlpqn}6a_?7Xo3|E#zHHV@KH)>kr+b^ zF^%Csj3L!{@aRWk2nVA#jWPN!c=Kw!aP{VUZ^b~2agzDX?o8*sw>xczRjf^x$&G4x zUmuc1A;AyurJ`O3fFPW11#r4SNH=KM9oWmTJF)B7%dxw#R|vP)jT`EnNg>67n!7VmXXeC)jB=%dW8Z*az%G_6Z5qZCgAZiWvZsFo?s1NmxV;afApEM~PbE7!f3n z6LrK1qMm3VLOljZkQ+j7iY7V}uhgX;=zy?49HJxH+f3a)CZ7*y2X^`$I246eUcI_{ z<80XU6xSv`q&sUjipl7UYb~Yqn;*X~_dMH3pKE-4`T3*iPt)(NZrCS~JlJJT4C;3x@oo7uL>mu~c724)Rdmr=f zuqDTyu*SB5g@#`S_`^G(FBwvNSP>102rQ@1JKvZ7+Ko-q|I*mdIS?(_UB9&S-8Ior3GjB{1I#m?8= z4Q4Qt$Sy3Ua?O*OOmiZeoi7CQsqEsd`MF^Gsg~B3HmfOsKPf!9aSdP?|Ga(ycAnIc literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04932_variant_null.parquet b/tests/queries/0_stateless/data_parquet/04932_variant_null.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0a6aaebe25a0ef7d6950feb4bf6d181b4f759dab GIT binary patch literal 724 zcmZ`%O>f#j5S?A9A(d5;C~NIXK5$4a4km?0L{U|w4yb9`QWcOW;^<-vB83FZSKCug zQBFCc{Gk4@&e)=D`Z2QK%+Bn*c_VEbyH%A{TDdK9um@oK5(sk%?kG_EO8d%E1^O-b zSeB%{qotkO-KA@f4>E)q4@5>%{}Z*>QZk{*X|4!>e=uYytr`B{$b!n6Rt>Z@AC99* zpd{3!7Ar*s`>E8AR>8$GO5^+_S_EOR2*9z7YkUt}Wt$pU1yd5M^rn)I<=hD8PI#}p z{6p===Y44cL4n*$CvPls3^GfyACmnXmGE5)=T>;%g|`vR`1>w@bK1W+of}f)<(-WL z<7ykb-H-hj>7M#tK4Uu>Az6ok>^^>lj8sRU+i z;#U3yHE%Iw{&(M4MUR2=JdA9}kcJ$k^FH@KRR;a;IDC_g!>>H= z3$j?!Gr-0=)k)RuKL6vR7Q}FJIuHAas-z{ISFeLEPJ?*1j27kLbXpGL_-b8TMe+Rn Y>azI!=&*8FHNFc#AJpl|w&^|p0QQ`Owg3PC literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/04933_variant_binary_and_string.parquet b/tests/queries/0_stateless/data_parquet/04933_variant_binary_and_string.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3fe6d010a1d7dcfa3996c0bf8e2e25ccb378486a GIT binary patch literal 802 zcmah|L2DCH5T1QW*KJ584Zg>Fu!kIU=^?F&k|L6VlVa5xF%8s^h`MMObg-bd`9V;fyOSMFrqHE0%bF3B;4$C15Q?G!wC!Jn6 z@TGu?(AG?*L1wQBy)yIrCt*@MgK+8x{?rHM7}pd%aAnq1z|NTsYnHqzg>uxnrA~$S z!SbB{u49U{0+1>}j#5?a8{2HtezeK5z;F3jyP3($E_|>957HkM4V%-i+hM&+>KjUt06w(<}Ts`&U9X6I^E@~ z7>*mnH>cAHLrsd0sSeg(Z;o#ENqVn06R(te7 zJbCnF;{RYg8&8A_(TkE`LX0QRoIDxd%u-4U@ek0+yf^dSoB6$ud7I7hTpv?diuF%1 zoxw!_Ymoo|MG7D(NI@hOi2*ZNXE&aUGnJ{1r3kI1i1Q{SPf3P>s*a=p(Mcfia)+Xi z4;p(-n_k{3Q!%ts3IwR8heia*v#4MGz2_;s97`CBIx;^_T*Tc2SG0 z!2t4_kad)UHE(6B={E0oT0M@-&^6U|LC3TNrtV`VNzL~zk+-Af57pgP+|R`MPdn#K)y!9#t=fb5@t>*O z$y>?dy`bh1ao>mCcbX{P`TcPg{$UH<+>M*MtC?>iU^|x1bGZj|cl_3|zSHYs)Onh7 zK7%C2J@u9+6)Vck6(zCFx2f=@0W%Tz=tn^_qTMl&PrlW1v$CjE0h(YI2V7=OAjpIesZ=_lgqtY9doMX)$p z@kmkR!+BwKyi%B(6QuV#G<4G?|vOXm~D(~k-eU8VN yw1Ab^@XDK8#cEG}W246_m)GlwTCrStyuOyWakDqo+h-3s0RDt;_(#d&59bfMq@2qD literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_parquet/05228_variant_malformed.parquet b/tests/queries/0_stateless/data_parquet/05228_variant_malformed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ff6e10db64008f1a63590e6fcdab4fcf5f623ae5 GIT binary patch literal 1001 zcmaKr&ubGw6vyApCfm5Cl7`N(BTLCamk4QVloXL5PGWwDv2FsT2E43ENY$oIlQaPj z9>u>yJb3c#(W8Hb2>uJ=+4p7}6B=;Xc{A_5na}&qgAH^mFJdAqat+}KNCRv(7yxjb z7S0&XI8FgVit9N%lM=SDJvT;CH%4bAWS6{+4Gdk)62M`W^`Q#LFNz0q{#J_a1S}q8 z=*$gHpobka6koqwXb-Y1aM}7Ymd~-*M7klkk-g@UTtpxsE#yKe0xS%>WDMkrsq1j& zpfdHM0Iogm9Q6jFh=JfYxyw$VfOblL68-ozJU;J5`^2C(3A^DW1nDSG6@BmoR|T+B z3hxOn9r=~XZ_E$;#Q&S5a@ZFvNP<(i6mQwe2L~TP%w)vOvT0P^kUul|h4~-3|DAbD zeOV#NO=YV+?EebvpM1l6C*L#wljZ;6gj?>?DspRqls_!#_zC&j0_C|$-D*8MydSmD z{mFM}%yda>-NZ@biPjkC>#Ej$UGAG$2@gD_U0LhLw4y&>(W7m-a}n3N{uZJ4(5>v( zOmIfFLI#oZT~v>3y^W?kCMjou(SvH5CSS+QLYk?sbuE9~tySN&pSO-odjBMnfo2; z&&8;i47izLI66D;O}0D3;dVG0oz61T-e~;l^dxiV?#}IQ)QxqO;DD3Jv0lMp0CVK|}{cAbvng0%DBGQYsilu|-FEE~r@#K56wjn?~nE28+JMZ)Nd$W`6sLFK+OPmpBF9=sa z13=~>0)Rnkl3JuTX@YbMsY5D2_@eP3<+~yw5@~6%S6W=>L)#k5^T@TWjhne>nomCb zBCJ=2Sj7t>rO{SUwovqw4N2@-4~()kn#xZXc5ntZB6^_}UC-?T=jL2JwquBhj&-vc z8^FTt0A1JMG82u8#mfDqBv+AYSe4SW0-;zmrKwAK7PuKW!QG%GZpu2!^Il9ZvM^mSIltL48LOfUvAk3RR4t$Yz`dH zsZqZ>{(C!ooe#%Tmh zQGH6OEGf%*rOK*rMx9e?M)l8VUmqo0`Xr0D{iQ#Cr3RO%=?O9iE)8mD znb!ecs3q;R9pmvjG)YT;o~g`@(!$|vR>yUHjT+(V{429lL*ut7OI0|(qL^&VryTPn zu(&_vFnJO^Nxapz&YzkXKQcLyo1Gqv=~28|nq@GWmnh`bSuTl#x>#k0b^jr4)u3Y3 z!Q{%r&e4YBPLNy(wOAcViLqe zrHYD5lsq~rDk>%_CRI$@0=)0-m?SYtGR7v-d*f%?s67AOk5Z=OCz54Nz)TPwogj)^ zd$=Gv=GlVi$Sv-!o^gAwsHn5m5=AABe!M3+{Ev=k0*GJ zSI}=nGNeFi#3B>2AvX%ZfQw=%jq<31ny8CLXpT1Mh;HbO0r&tTFcymh#yvd7UwDaxG)#&&kqYUM5$_@w@}meG6h$eNLuJ&! zduWJeXpIi&ieBiC_c0t}FcH)6A?9NVR^VfNf-TsIeK?4tIDs?x7ME}p*YO+f;t`(V zA0$W^74;gDBPG(}9b`pLK7$0lsY9(;~3a2%&_9^c~!{ET1m z2Oi)lULcB-KQUg%TS$ZS$c*gBgMu*Op*Z4D0aZ~8_0SkC&=#H09epqmLogEKFc~v2 z2Me(jtFR6mu?@TN84lwZPU0Lc;0mtcCVt0#Ji&9kLZT#`|44zU^h5#7)m1MmSxU@RtKI%Z=5KEg_@#i!VcUD%I9_!3{^EWX2K{Dd30 zjeB^EzwiA}qsdtj8v7#~yr+FK`^Ea30^|2mFj*@dqB@DPBNd_!HxGyoEGKkIcx9JSYeg z9*QFl6;KtmP!EmK0&UR=-O&dFF$5zq4wEqhbFdIgu?p+35!`#}hoqD=fJj9>)8__8^|M3RiMhr3_3vwVY z3c-Sp5-5X;sD|38k0xk|cIb>A=!-!ZicuJkDVT}5ScGL*jrG`s?bw6Q@db|K6wc#& z{D7bFEB?R(JjDy>MSo(vj<=8o>5&=Pkp~4~!b5Syp#rL+7V4ofTA(dDp*#9uAckNh z#$hsMU=9{yDOO<}Hewrg<1-w_F`UFXT)-7v!%h5-`*?!qc!fl7bN(X*QX>|bkPW#} z00vwXLur&p71TssG(vNS?=zy;1h5mRS z!!ZUEF%2JLK9*nwKE@~5f}PlhgE)#4ID>C-30H9)zu_((;Tirx0=|=d4at!bY4HxS zA}8{pFl+=UiL$7K>ZpSTXo^;7k1ptmei)2l7>x;-idmS4#aNCt*nrL0fxS3@BlrrZ z@eMBGNBn|YxPyoI6MrK*HRnIxz}tvH24q1FRM1j2W1Ng;6_!?5T48I2trfOb*jiz0g{>8~ zR@ho$YlW>9j#fBY;b?`U6^>RoTH$DgqZN);I9lOqg{u{=R=8T>YK5y6u2#5O;cA7e z6`odjTH$Ghrxl)7cv|6Ug{KvsR(M+BYlW{BzE=2J;cJDj6~0#ZTH$MjuN8q-1X>Yj zMW7XdRs>oRXhonEfmQ^pFx1>ob3@GyH8<4UP;*1g4K+8^+)#5v%?&j-)Z9>WL(L5} zH`Lrvb3@GyH8<4UP;*1g4K+8^+)#5v%?&j-)Z9>WL(L5}H`Lrvb3@GyH8<4UP;*1g z4K+8^+)#5v%?&j-)Z9>WL(L5}H`Lrvb3@GyH8<4UP;*1g4K+8^+)#5v%?&j-)Z9>W zL(L5}H`Lrvb3@GyH8<4UP;*1g4K+8^+)#5v%?&j-)Z9>WL(L5}H`Lrvb3@GyH8<4U zP;*1g4K+8^+)#5v%?&j-)Z9>WL(L5}H`Lrvb3@GyH8<7VRC80!O*J>w+*ET@%}q5o z)!bBbQ_W2^H`Ux!b5qStH8<7VRC80!O*J>w+*ET@%}q5o)!bBbQ_W2^H`Ux!b5qSt zH8<7VRC80!O*J>w+*ET@%}q5o)!bBbQ_W2^H`Ux!b5qStH8<7VRC80!O*J>w+*ET@ z%}q5o)!bBbQ_W2^H`Ux!b5qStH8<7VRC80!O*J>w+*ET@%}q5o)!bBbQ_W2^H`Ux! zb5qStH8<7VRC80!O*J>w+*ET@%}q5o)!bBbQ_W2^H`Ux!b5qStH8<7VRC80!O*OaF z+){H(%`G*z)Z9{YOU*4cx76HHb4$%FHMi8HFwqARdZL(T{U;r+*NZ|&0RHj z)!bEcSIu2Dch%fgb63q>HFwqARdZL(T{U;r+*NZ|&0RHj)!bEcSIu2Dch%fgb63q> zHFwqARdZL(T{U;r+*NZ|&0RHj)!bEcSIu2Dch%fgb63q>HFwqARdZL(T{U;r+*NZ| z&0RHj)!bEcSIu2Dch%fgb63q>HFwqARdZL(T{U;r+*NZ|&0RHj)!bEcSIu2Dch%fg zb63q>HFwqARdZL(T{ZXA+*5N;%{?{u)Z9~ZPt83w_te}|b5G4ZHTTrqQ*%$vJvH~# z+*5N;%{?{u)Z9~ZPt83w_te}|b5G4ZHTTrqQ*%$vJvH~#+*5N;%{?{u)Z9~ZPt83w z_te}|b5G4ZHTTrqQ*%$vJvH~#+*5N;%{?{u)Z9~ZPt83w_te}|b5G4ZHTTrqQ*%$v zJvH~#+*5N;%{?{u)Z9~ZPt83w_te}|b5G4ZHTTrqQ*%$vJvH~#+*5N;%{?{u)Z9~Z zPt83w_te}|b5G4ZHTTrqQ*%$vJvH~#+*5N;%{?{u)!bKeU(J0r_to52b6?GUHTTur zS94#@eKq&h+*fm7&3!fZ)!bKeU(J0r_to52b6?GUHTTurS94#@eKq&h+*fm7&3!fZ z)!bKeU(J0r_to52b6?GUHTTurS94#@eKq&h+*fm7&3!fZ)!bKeU(J0r_to52b6?GU zHTTurS94#@eKq&h+*fm7&3!fZ)!bKeU(J0r_to52b6?GUHTTurS94#@eKq&h+*fm7 z&3!fZ)!bKeU(J0r_to52b6?GUHTTurS94#@eKq&h+*fm7&3!fZ)!bKeU(J0r57azR z^FYl5H4oH0Q1d{|12qrSJW%sM%>y+L)I3o0K+OX+57azR^FYl5H4oH0Q1d{|12qrS zJW%sM%>y+L)I3o0K+OX+57azR^FYl5H4oH0Q1d{|12qrSJW%sM%>y+L)I3o0K+OX+ z57azR^FYl5H4oH0Q1d{|12qrSJW%sM%>y+L)I3o0K+OX+57azR^FYl5H4oH0Q1d{| z12qrSJW%sM%>y+L)I3o0K+OX+57azR^FYl5H4oH0Q1d{|12qrSJW%sM%>y+L)I3o0 zK+OX+57azR^FYl5n)638MKlZzQ^V4*H5?6B!_)9Jw1Vagt)MwWD`?Kp3Ys&tg60gZ zpgBV;XwJ|InlrS5<_xW%IYTRG&d>^)Gqi%{46UF!Ln~;`&^)Gqi%{46UF!Ln~;`&^)Gqi%{jNsp12j}uf z*#G8gw=F9Z#2mW(uUEUxpBbGorZRt`9$xOI>%XG79x00Jc=&_-$DMO>(MS{*TSRe@ zLKJ_wAN4PPDIUdNz((;Gmr?wMU(~<+MOhSoDHFwCX6UlKsDJsxbFR$eZ!?e%vC!3X z8SoA=A`^5qT^3};yU2#@$bp>5h1|%4yvT?AD1d?}gu*BS112okaG>7>U7hD6Kv5Kf zuGTAok|>4Jh(j5aMLCp51yn>O@LLsC1yxZE)lmaAQ46(E2k)US>Y+XwpdlJTR|7Ud zQ#3?&yJ@=!M?sgTCm8{ulsVZ8!*n@jgC) zu09-!VHl1P7>Q9BjWHODaTt#Yn21T3j47CkX_$@~n2A~V5VJ7{b1@I|u>cFP2#c`< zA7Lq$VL4V{C01cI*5G5T#X79V27H1~u@RfF8C$Rw+prxwuoJtm8+))9`>-FM;d30o zK^($ie1Ri4iZ5{t$MF?T;A@=3DV)X`oW(hu$2a&E-{Atj$3Th(~ygCwPiK@eF_AIbPsz{DYTxg{Wx$hXhCn zT`igzN$?tyLRXKzjyIr7Z{LKjCVdNUBPCKnSC^(i4ALSUbhT-EWWYPf2wi=e8Cj4O z??P9jW=9U>L@wy+)I7+Ge8>+yTNOkh6h;x~>QxgKY&g)>tR8#>C<fQqPu%FxxYRZ$JqQ3JXy-wcOV8~Fy4o*)*XVO z7>3~(fsq)6(HMiV7>DtgfQgud$(Vwvn1<22aFc)aV+}sWTCBr*Y``b@6dSP#o3RC3u?^d?13R$`yRip*u@C$489v7W9K<0U z#uqq(qxcfXa2#LZ1ir>eoWg0G!C9Qcd3=L!@f|MUdtAgNT*ejrfFE%cKj9jF#xJ;z z8@P#IaSOlUHh#w+xP!a6hx>Sdhj@g?c!H<+6VLD$p5q1n#y@z8SI}2}E{Tsy09{?3 z2#JvduR&K^C&TM_1IeMQuT$VHyp5F5)!3<#1~EtrU7a0^^vHmBpsTerAv3ZdD|GdC zHe^Q*~&(x}yhrLRYW%Mj!M=Kj>=q0T_ru7z|zA{sD$yD28D; zMqngHVKl~IEXH9xCSW2aVKSy*DyCsNW?&{};X};E9L&W$%*O&O#3C%l5`2WEScc_T zft6T=)mVd%u@>vF9vkopKE+0C!e(s2R&2v|?7&X!!fx!rUhKnue1^|)00(ghhw%lD z;3&StF&xKNIDxNm5~pw)XK)tha30^_TYQHL_#PK=372sNKj24P#ZS0~pYaQ>;|6Zx zSKPvHxQ*ZO2kzi5?%_Tj;2|F2F`nQl{=_rswpxq4V5R3H40PSwbh)l?gEXay?kqz0A1KRzN z3%QX8d7<49`B4A`Q3%={Q3M7|SkP_>2QEDL(C&$%D2C!F0qv$Jh0=&a8I(milt%?r zgmzn0Mio>=HE8!m4b(&})P{Cryob7|hx*X&jD~20#%KcV)@X+2Xn~e!h1O_;wrGd; z=zxysgwD|Jj;`p2?&tyS_UMJ)=!3q{?vMT$fPokU?FM-tA7BWELc2qTV+2NG6h>nV z#$p`CV*(~(5+-8`reYeVV+Lko7Cyvm%)wmD!+b2jLM*~!EWt-uie*@i6#+f!;8SeGCTzwQY{fQg#}4eoF6_o0?8QFp$7lE)2XGLFa2Q|U2#(@Q9K&&Z zg%kK1Cvgg=aRz5`4(IU=zQuRAfbVe;mv9+Z@B@CtRs4i&_!+<8I&R=5e#I^PhTHfZ zf8Y-8;vVkf0UqKJ9^(m~;!ixoUwDoe_#6M=C0-#a5$8V=AR!VVF|?cKH6%qcybkT| zNsc#>0&hXPeNrM7QX>tt`zI~ZAr|SO-9YakBQhZ~v^yv(-bFTKhjt6)L@wk;9%%Pa zKIBIM6ohsY6-E&lFrnQ=HXOL{pxs6RilP{bL%WYkq7+Ia4rNdlcn3L95H3og5^AF{+Mp}?VJOBy-!JR?;pNcxv-*Bj-;e71O?^M9?-#E@-|y-B zIeovDnAd43kPcaq7bc3K9IB%}TA(9(VKDT1Ua!~ndR(u!^?F*bm-TvBuXpu&RBXXh;Jd{Rd)Ik%pMK|=vFzEGyUJvN&y}q97>$Scf>+7w) zp6ctRz8>o9oxYyw>y^G9CE;`NEyUtoKg$K~?uY^fBUlD`M&~C3H2v7!9q1|20&>lT75ZcW(5woxetDxOm zJ8=NVaSqz8bscx`6tAG&St*eL*--%6ja3pAQ45Wr-B(@E7eg=>+HExt%diexq1{zS za0(aj6SSM^AzmO+Ql5{X-BVeR2L_5lyQQk39-5;Av^#1LMq&~^gmy!%#zySIL1_2W zd0fE_+=X^K>Eus_RCov4-Bb`RN}&?8o2fC{pey=8yO+jcI_6_Jv|DK#_Twl{L%Wl% z;Wi%OZ)i7C3Zz3;UU_sWl^W=kt;wTU8-f4iA=!D+TZk^GXf;spI+MTl*dvF+EL%VT)#ILxIzo6YW zZy*gaAs4jU#z!2g;5}$}O*?eQ01StA)6B#|ti&hK?wQYV3}B6;Klm(F)rA(gz=445mW6U6x`kwqP%`yX7Rl!&TgZcC$Q3!nZj85d-aB z$&DfiPzKtqQWwq89zCGlDI+iuv#uJu(gRung;<-6H#N1gCHT+8y#69^wTOrR4laT4X^U7|`yI zvZ#i7Xb$c6=!roXiAm7zj>TAwjo1b4=J*QdaRoP^-5Y;GU-*+D6|`F;2MWSPDQI^_ zZ8Sz3bcJ?f48=H1$9!n_#d>VRejJ5%TYQgexQ$29?ux{zIscIkS)tt&CW@gPszbXc zTA(9(VKB5?Vlrl93D!WnBX;8uPT(78H^fcc!!tyu;rvHxWJFFBf_6KUMrG7N6KHos zH}uCajE8nJEWiqEz;;?Qn{8fbu)=mhOP zcpsxN1#|EbKE`J3!C`!jZ}B63#eMvR1Zg?{kp`KN3x(k$4ps0TnxY-LV*rL@0%l?% zR^k)vz~?xIv$%v`@CTmYC0H#3X!(#aNAv*oA}m3g>YJH*gn!LNEH0Ar;<14itooQmBO5 zXpA=KihdZ1ahQ(zSdR7BhW$8-)A%0Oa2t>CHxg&${6{)uMP8UFhH|Km`e=cU=!L-; zg~^zWC0K(^*o{Ltfp72wZsH!EAvzQ1KT;zja-tACltyLLK@+q^H}uCajK>TtzzS@@ zc6^2}aRwLhGk(Wo{DUN!IsXxhcaaYkilaPgpaEK<6MEx)jK&nq!AJNQo3RIn@io51 zkN6e$@fQ+g;rvG$WI`?!hL1Q@!Fy)^;iG^5+Pp|`@;~3835`MuSc!HOB zEi30g(jy!4!$t{IKut76D|AL5e1I{Sin&;dwb+8a_yQ;K9j@XQ9^g3=zRUTK7-U9n z6hVM8sEWF1hW6-zff#{_n1w}Hg-@{)2XGwca2eNe2T$<|NwabOBLlLd034J=Mbttg zv_=>7#So0eG|a;?tix99!x5aq1^k5H@DMMMC_Cpr(jp7;z(7%yMK#nzb96vY48llE z!iQLl)!2w#IEb%s9#?P!ckw6m4SzDE!aK-;f^bm^l~5av(FR@74?{5y(=i{*u^!v7 zA4hQ--{Tr?;}QNw;+&lSNQbP*3lqgq4%JZ~Ezl9YFc_mS8MCnjYp@BsaR?{y4Sv8) z+`}_O=i>ZFYGg!C6oQA+sEj&jg0|>}{uqYwn1KaYfeqlxLi}57bmSmGAW`HX zapWLL<3CADs@-+T{*NyQ9 zdO@eRSYBm?Z?U}mi9g^qN&EpX7UB>13LbyJm)7`$$Ojh77ozz0=qrTYVtLvQ-(q<_ zjX&T?F8+XLpZEiw65BywOx4$R1b6*;gY2TtU`jU0H913z*QL=K8Z4vIw% ziboDgL=H+u4oXE1N=FXjA_rw62W2A%P8OgMGopm4jM!b8b+lWo-=7wY@=jJ6Gg?Q$eklP zI#Fzs=p=DXxK1Dm|ENl{a5fKTi*U9KXRC0w4riNiwhd>yaJCO;hj4ZbXQyy>4rdq5 z6eGg>>c(*@t|icY^$2IraP|sk?{M}BXJ5?}nPc<_`*WO%s|&PhU^oYbKl^2Po9~A| z`$72rA>kYv&SBvk9?lWr92w40;T#>#G2t8=&T-)!AI=GyDbnTCy-(se6@RX&d!G`{ zso|U!>Qt5zd+6oE6Rw!#O*gbHX_{ob$ptKb#A~xll7jukgMWbDWAlJJo%C6wamL zTo%sd;am~UmEl|!&eh>u6V8vrxi*~ZG*j#gZ+Qd9sko9sxBO{1H->XlI5&rLOE|ZN zb6YsKhjT|bcZPFUICqD0PdN96b6+_3hx4;=ejd&P;XD}5L*YCe&M(4wB%DXX`DHkd zh4XkgzY6DxaDE-mli@rS&eP#M6V9{YJQvRMnkgpaQoCBMoAZ7U>X+ z^vHmBkP(@X8Cj4O?;;zrBL{LK7jh#H@**GdqW}t`5DKFR44ANB!+{GAJ^~a)F&_1d zJo>#V!J{t8drF}+;!p-r+Fc5<< z81M6l&*s(d!4MwtP~I~P!!ZIQF$$wG24gV}<1qmfF$t3~1yeB%(=h`xF^hYsoKN>K zn|qkUd*)&u=3@aCVi6W&2|mJ7EW>iFz)GybYOKM>Sc`R7j}7<)pJF36VKcU1E4E=f zc3>xVVK?^h=#uBxqua-$+s}JG!{<1FgE)l4_yR|86kp;Pj^itwz}Gm5Q#g$?IE!;Q zk8kiTzT+O}h4=V9_jr-_T*75s!4LQmSMd|B;b;7U>$riN_!YPC8*bxw{DC{Ti+i|_ z2Y84_c+5Q)Dxi<`Dfj#*?|Fv5@EkAjH~zs(yh@oYRdYK9f3*;uqJ#Z^_oNW@7B}=Z zQX&;nBMoAZ7U>X+^vHmBkP(@X8Cj4O?;;zrBL^Qw=7Rb#a`9p0<~@0k7x|DM1yB%$ zP#8sEz=Q=G4!6=LycLgI@p(^xq9}&qD1nkFh0=&a8I(milt%?rL?u*46;wqvR7VZe zL@m@t9lVFSsE7J!fQD$qBg$P!A6F9|QB&U249(F3Ezt_C(FSeN4(-ta9nlG$(FI-6 z4c*ZLJ<$uj(Fc9e5B)I!12G7L@jgDl5DdjI9@&lXk&WPyjpRL}FdAbp7UM7;6EG2z zFd0)Y71J;sGcXggxW&qa^|AgJeNu?p%cITh(~ygCwPiK@eF_AIbPsz{DYTx#nVCK67f$5ZA$#tdOG-ze1-6z^;GbG z`syG>8%w8C2Tsuc;HQHB*RKxvo33P0voQyAF%R>x01L4Qi?IYBVJVhjIaXjLR$(>P z;A5=CI;_VAe1cE05u30XTd)<|upK+F6T7e*d+?v|G>~GkqmmaX`Cs*kpniq;CxR9g z{{Iuf|ITZIjFAhp{=F@63Anl`CQBi&u=UKp@%m#b5(-0tFJ`oy7??Y zuHlRS7q_%-Wdd%AJJG{R`|M!Cv!@s8TxI>2A4mSnk91hJ&%eAsF0SHW zZq(<8;LtkalhfOZ``qU zrTSg@RdECL>j%c^!NxzH0r8KMJBz=4evKYHZ{bH0e;EEa{PnFH)GyMgL7k3mE5<)O zem=6aw(g9Nmxs_MuAG*{|75s0u6E_{_A7-y`q#UPzt@Vk-plO|(0YDJc-MMgnea~I zN|cC-&Dyh7?_T}d^)1+@XU~GIdiU-zAZ!12z58_P(LJka6)IB5NMD>kgpU7Tcnx?h I{vQ?o8_YuG7=CBiu8^z=w3*FpgN8)A!K6j1r7;anI$(vyufn0ErUw^Fz@)SY6o?1o z(RlXg(Recc0RM#-&;ABa#*^q+lS1}eBwy=}ZqNuShq%U}h1SVoBj+QgV0nzeeD@404mof1z0x`Bj2o}x-oI#MO z*Z(Kkg!D3&nlV)^FUidi@rH@#p^6=uN6^M|fWT%irI~3NDlUgE? z!dU?s?KB6$aWDwT<#J&doSZkoRo=@`s38&23Y_*?G2zTqQV zK15S&kd)3o8cJ6xI^2Pasi##BF6B-ZE|8J$RG@Ue!(=OGX~%!`2Fs;dxO)L*1tqlhnK7YUWHtm8ck4sCwkj0V|5-tObSqQxd zthdJk-tU6d@8)bs#Oa4d&}67gN^=6I0#i+6C4!hIP=S|I0dJ~KjJ zm+K|z8uW==ABE5YWZa5ZE3Fg`TxWoG+>UE4j+gFJ`EMC(f0CJEnZ92d_?#GZVL4Uu zTPeohaf1DT@;#Ph$^b7N`xImxkkjk~0vR%3_P(rF8;z|iIz}{ri*yz_MJ#*YJ}|R# z3|)XB?7?f(tY6rSQ5T9I2qbF04A1sN)clJed`wu1Y?mc%XAYzlJf8&LX;tdgjaXXb zxWRnfgh|qS+7@KH8`XBaMk`88doOP6R>2cN7~~=dsmUZb>454a^Mv=7c|R~}MDG86 zxFAbQ1Id6?nvUE}Cx}4MYBYvOFfzDz%Eo|wMeiF%hdg@C!V5mlxyD{8aDpiYsdK%P z(M;nb<{h!z7{H7zf#3l)O4E6*~jz7+0lAdD{86K zjIk3ut&7W~3N1}#XTk6xHj5^1>QAW=uE}4iuNI#+s7Q_L{JS%x4Bo5(A(A8sp1Eq! ze0$|txxO+#H@2K0Q+&}3t`!YSLoBO&Cz;a(O)W+1=5jXQiJ#6@-I|7)G4&O_pRQ0x y67Yw&Q)QxqO;DD3Jv0lMp0CVK|}{cAbvng0%DBGQYsilu|-FEE~r@#K56wjn?~nE28+JMZ)Nd$W`6sLFK+OPmpBF9=sa z13=~>0)Rnkl3JuTX@YbMsY5D2_@eP3<+~yw5@~6%S6W=>L)#k5^T@TWjhne>nomCb zBCJ=2Sj7t>rO{SUwovqw4N2@-4~()kn#xZXc5ntZB6^_}UC-?T=jL2JwquBhj&-vc z8^FTt0A1JMG82u8#mfDqBv+AYSe4SW0-;zmrKwAK7PuKW!QG%GZpu2!^Il9ZvM^mSIltL48LOfUvAk3RR4t$Yz`dH zsZqZ>{(C!ooe#%Tmh zQGH6OEGf%*rOK*rMx9e?M)l8VUmqo0`Xr0D{iQ#Cr3RO%=?O9iE)8mD znb!ecs3q;R9pmvjG)YT;o~g`@(!$|vR>yUHjT+(V{429lL*ut7OI0|(qL^&VryTPn zu(&_vFnJO^Nxapz&YzkXKQcLyo1Gqv=~28|nq@GWmnh`bSuTl#x>#k0b^jr4)u3Y3 z!Q08pG7q0UZ^;fSElz4hdrAo!yz)`Q~9=-8A#U68aSG4}1yJ6o)9)T90*U+{SL_4|C?})` zsQbs=aq=b^C#iaCk{q3PLHat<#TX*NbpgCkW1Qg9m!FvY%;H+?|9>2M4a>)b)N+VN z@xG^H@UaDA1}QVgCYRYQ{2h}YSp18{Us!Z)HaCDW(4OAroc@!)nQ03acp=}j_&1Ng zapElob(7{VX!4^Y{n(dZ9ng^qOtbmo)#G%C>N|fxYidfymHVV$3Y)KLlVFj!(tP$xNlWAQ>t?^bx4YR&Go!y@ z8z+IyViPS>nx=hPr;SB#-(D8|&Fy#|5*nMT*!T|3XSS-=11dX6hG*y9@p@-4SWkw- h(`k0n9gg0fo@5_xZESAjbuj`+fAAAOi#mSHe*kK+rjq~w literal 0 HcmV?d00001