From 679a896b3c9a7503ec5fa323f1d4f4d78d09fa8f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 6 Oct 2026 16:53:14 +0000 Subject: [PATCH 01/81] Backport #122718 to 26.8: Coalesce adjacent singleton row-mask updates during part mutation --- src/Storages/MergeTree/MergeTreeSettings.cpp | 32 ++ src/Storages/MergeTree/MutateTask.cpp | 379 +++++++++++++++++- .../__init__.py | 0 .../configs/background_profile.xml | 3 + .../configs/expanded_ast_profile.xml | 3 + .../configs/expanded_ast_users.xml | 7 + .../configs/limited_profile.xml | 8 + .../test_mutation_row_mask_set_limit/test.py | 358 +++++++++++++++++ .../05256_coalesce_row_mask_updates.reference | 12 + .../05256_coalesce_row_mask_updates.sql | 84 ++++ ...05257_coalesce_row_mask_subquery.reference | 3 + .../05257_coalesce_row_mask_subquery.sql | 34 ++ ...esce_row_mask_separate_mutations.reference | 3 + ...8_coalesce_row_mask_separate_mutations.sql | 51 +++ 14 files changed, 976 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_mutation_row_mask_set_limit/__init__.py create mode 100644 tests/integration/test_mutation_row_mask_set_limit/configs/background_profile.xml create mode 100644 tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_profile.xml create mode 100644 tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_users.xml create mode 100644 tests/integration/test_mutation_row_mask_set_limit/configs/limited_profile.xml create mode 100644 tests/integration/test_mutation_row_mask_set_limit/test.py create mode 100644 tests/queries/0_stateless/05256_coalesce_row_mask_updates.reference create mode 100644 tests/queries/0_stateless/05256_coalesce_row_mask_updates.sql create mode 100644 tests/queries/0_stateless/05257_coalesce_row_mask_subquery.reference create mode 100644 tests/queries/0_stateless/05257_coalesce_row_mask_subquery.sql create mode 100644 tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.reference create mode 100644 tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.sql diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e44eb4a2aaad2..5502513e81deb 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -548,6 +548,38 @@ Limit the number of part mutations per replica to the specified amount. Zero means no limit on the number of mutations per replica (the execution can still be constrained by other settings). )", 0) \ + DECLARE(Bool, enable_row_mask_update_coalescing, true, R"( +Coalesce adjacent singleton `UPDATE _row_exists = 0` commands while mutating a +Wide part with Full storage. The persisted mutation commands and their +versions are unchanged. +Set to false to execute each command separately. +)", 0, \ + {"26.10", false, true, "Enable row-mask update coalescing by default"}) \ + DECLARE(UInt64, max_row_mask_update_coalescing_keys, 256, R"( +Maximum number of keys in one coalesced row-mask update. The effective limit +also respects `max_rows_in_set` in the background query settings. Values below +2 disable coalescing. +)", 0, \ + {"26.10", 256, 256, "New setting"}) \ + DECLARE(UInt64, max_row_mask_update_coalescing_key_bytes, 64 * 1024, R"( +Maximum total number of key-literal bytes in one coalesced row-mask update. +This bounds the size of the generated `IN` predicate. Set to 0 to disable +coalescing. +)", 0, \ + {"26.10", 64 * 1024, 64 * 1024, "New setting"}) \ + DECLARE(UInt64, max_row_mask_update_coalescing_commands, 4096, R"( +Maximum number of part-local mutation commands eligible for row-mask update +coalescing. This bounds the extra command-list copy made by the optimization; +the value is a resource guard, not a semantic limit. Values below 2 disable +coalescing. +)", 0, \ + {"26.10", 4096, 4096, "New setting"}) \ + DECLARE(UInt64, max_row_mask_update_coalescing_ast_bytes, 1024 * 1024, R"( +Maximum sum of the original commands' `ast_text` bytes eligible for row-mask +update coalescing. The command-count limit alone cannot bound this copy when +commands contain large literals. Set to 0 to disable coalescing. +)", 0, \ + {"26.10", 1024 * 1024, 1024 * 1024, "New setting"}) \ DECLARE(UInt64, max_number_of_merges_with_ttl_in_pool, 2, R"(When there is more than specified number of merges with TTL entries in pool, do not assign new merge with TTL. This is to leave free threads for regular merges and diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 0e5d228f1508e..0ada3db421781 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -2,6 +2,13 @@ #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -9,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -79,6 +88,10 @@ namespace Setting { extern const SettingsUInt64 min_insert_block_size_bytes; extern const SettingsUInt64 min_insert_block_size_rows; + extern const SettingsUInt64 max_rows_in_set; + extern const SettingsUInt64 max_bytes_in_set; + extern const SettingsUInt64 max_ast_elements; + extern const SettingsUInt64 max_expanded_ast_elements; } namespace MergeTreeSetting @@ -103,6 +116,11 @@ namespace MergeTreeSetting extern const MergeTreeSettingsBool propagate_types_serialization_versions_to_nested_types; extern const MergeTreeSettingsBool share_nested_offsets; extern const MergeTreeSettingsMergeTreeMapSerializationVersion map_serialization_version; + extern const MergeTreeSettingsBool enable_row_mask_update_coalescing; + extern const MergeTreeSettingsUInt64 max_row_mask_update_coalescing_keys; + extern const MergeTreeSettingsUInt64 max_row_mask_update_coalescing_key_bytes; + extern const MergeTreeSettingsUInt64 max_row_mask_update_coalescing_commands; + extern const MergeTreeSettingsUInt64 max_row_mask_update_coalescing_ast_bytes; } namespace FailPoints @@ -130,6 +148,305 @@ enum class ExecuteTTLType : uint8_t namespace MutationHelpers { +/// A sequence of row-mask updates with one key per `IN` predicate has the same +/// final mask as an update over all matching keys. Equal prefixes use one `IN` +/// predicate; mixed prefixes require an `IN` predicate over prefix/key tuples. +/// This is only used while processing a part; the persisted mutation commands +/// and their versions are left intact. +struct SingletonRowMaskUpdate +{ + String prefix_column; + Field prefix_value; + String key_column; + String key_value; + bool key_from_uint128_array_join = false; + size_t predicate_ast_size = 0; +}; + +static bool containsAliasedExpression(const ASTPtr & ast) +{ + if (!ast->tryGetAlias().empty()) + return true; + + for (const auto & child : ast->children) + { + if (containsAliasedExpression(child)) + return true; + } + + return false; +} + +/// The targeted singleton `IN` source is `SELECT toUInt128(arrayJoin(['id']))`. +/// Accept only that exact one-column, one-expression `SELECT` shape. In +/// particular, a `FROM`, `WHERE`, `LIMIT`, `UNION`, or different cast is not folded. +static const ASTLiteral * getSingletonUInt128ArrayJoin(const ASTPtr & rhs) +{ + const auto * subquery = rhs->as(); + if (!subquery || subquery->children.size() != 1) + return nullptr; + + const auto * union_query = subquery->children[0]->as(); + if (!union_query || union_query->children.size() != 1 || !union_query->list_of_selects) + return nullptr; + + const auto * selects = union_query->list_of_selects->as(); + if (!selects || selects->children.size() != 1) + return nullptr; + + const auto * select = selects->children[0]->as(); + if (!select || select->children.size() != 1 || !select->select() + || select->recursive_with || select->distinct || select->group_by_all + || select->group_by_with_totals || select->group_by_with_rollup + || select->group_by_with_cube || select->group_by_with_constant_keys + || select->group_by_with_grouping_sets || select->order_by_all + || select->limit_with_ties || select->limit_by_all || select->limit_after_all) + return nullptr; + + const auto * projections = select->select()->as(); + if (!projections || projections->children.size() != 1) + return nullptr; + + const auto * conversion = projections->children[0]->as(); + if (!conversion || conversion->name != "toUInt128" || !conversion->arguments + || conversion->arguments->children.size() != 1) + return nullptr; + + const auto * array_join = conversion->arguments->children[0]->as(); + if (!array_join || array_join->name != "arrayJoin" || !array_join->arguments + || array_join->arguments->children.size() != 1) + return nullptr; + + const auto * values = array_join->arguments->children[0]->as(); + if (!values || values->value.getType() != Field::Types::Array) + return nullptr; + + const auto & array = values->value.safeGet(); + if (array.size() != 1 || array[0].getType() != Field::Types::String) + return nullptr; + + UInt128 parsed; + if (!tryParse(parsed, array[0].safeGet())) + return nullptr; + + return values; +} + +static std::optional getSingletonRowMaskUpdate( + const MutationCommand & command, UInt64 max_combined_key_bytes) +{ + if (command.type != MutationCommand::UPDATE) + return std::nullopt; + + auto alter = command.ast(); + if (!alter || alter->partition || !alter->predicate || !alter->update_assignments + || alter->update_assignments->children.size() != 1) + return std::nullopt; + + const auto * assignment = alter->update_assignments->children[0]->as(); + const auto * assigned_value = assignment ? assignment->expression()->as() : nullptr; + if (!assignment || assignment->column_name != RowExistsColumn::name + || !assigned_value || assigned_value->value != Field(UInt64(0)) + || !assigned_value->tryGetAlias().empty()) + return std::nullopt; + + const auto * predicate = alter->predicate->as(); + if (!predicate || predicate->name != "and" || !predicate->arguments || predicate->arguments->children.size() != 2) + return std::nullopt; + + const auto * prefix = predicate->arguments->children[0]->as(); + const auto * membership = predicate->arguments->children[1]->as(); + if (!prefix || prefix->name != "equals" || !prefix->arguments || prefix->arguments->children.size() != 2 + || !membership || membership->name != "in" || !membership->arguments || membership->arguments->children.size() != 2) + return std::nullopt; + + const auto * prefix_column = prefix->arguments->children[0]->as(); + const auto * prefix_value = prefix->arguments->children[1]->as(); + const auto * key_column = membership->arguments->children[0]->as(); + const auto * key_value = membership->arguments->children[1]->as(); + const auto * subquery_value = key_value ? nullptr : getSingletonUInt128ArrayJoin(membership->arguments->children[1]); + if (!prefix_column || !prefix_column->isShort() || prefix_column->isParam() + || !prefix_value || prefix_value->value.getType() != Field::Types::UInt64 + || !key_column || !key_column->isShort() || key_column->isParam() + || (!key_value && !subquery_value) + || (key_value && key_value->value.getType() != Field::Types::String) + || prefix_column->name() == key_column->name() + || prefix_column->name() == RowExistsColumn::name || key_column->name() == RowExistsColumn::name) + return std::nullopt; + + if (containsAliasedExpression(alter->predicate)) + return std::nullopt; + + const auto & key = key_value ? key_value->value.safeGet() : subquery_value->value.safeGet()[0].safeGet(); + if (key.size() > max_combined_key_bytes) + return std::nullopt; + + return SingletonRowMaskUpdate{ + prefix_column->name(), prefix_value->value, key_column->name(), key, subquery_value != nullptr, alter->predicate->size()}; +} + +static bool hasStableRowMaskKeyTypes( + const SingletonRowMaskUpdate & first, const StorageMetadataPtr & metadata_snapshot, const ColumnsDescription & source_columns) +{ + if (first.prefix_value.getType() != Field::Types::UInt64) + return false; + + const auto prefix = metadata_snapshot->getColumns().tryGetPhysical(first.prefix_column); + const auto key = metadata_snapshot->getColumns().tryGetPhysical(first.key_column); + const auto source_prefix = source_columns.tryGetPhysical(first.prefix_column); + const auto source_key = source_columns.tryGetPhysical(first.key_column); + if (!prefix || !key || !source_prefix || !source_key + || !prefix->type->equals(*source_prefix->type) || !key->type->equals(*source_key->type)) + return false; + + const auto type = prefix->type->getTypeId(); + const bool valid_prefix = (type == TypeIndex::UInt32 && first.prefix_value.safeGet() <= std::numeric_limits::max()) + || type == TypeIndex::UInt64; + const auto key_type = key->type->getTypeId(); + return valid_prefix && (first.key_from_uint128_array_join ? key_type == TypeIndex::UInt128 : key_type == TypeIndex::String); +} + +static MutationCommands coalesceSingletonRowMaskUpdates( + const MutationCommands & commands, const ContextPtr & context, const StorageMetadataPtr & metadata_snapshot, + const ColumnsDescription & source_columns, UInt64 max_keys_per_command, UInt64 max_combined_key_bytes) +{ + if (commands.size() < 2 || max_keys_per_command < 2 || !max_combined_key_bytes) + return commands; + + /// A larger `IN` set must not start throwing or silently truncate where each + /// original singleton set would fit the effective background settings. + const auto & settings = context->getSettingsRef(); + if (settings[Setting::max_bytes_in_set]) + return commands; + + auto max_rows_in_set = settings[Setting::max_rows_in_set]; + if (max_rows_in_set && max_rows_in_set < max_keys_per_command) + max_keys_per_command = max_rows_in_set; + if (max_keys_per_command < 2) + return commands; + + /// The analyzer counts expanded QueryTree nodes, including source + /// expressions retained by folded constants. The raw mutation AST size is + /// not an upper bound for that limit, so keep the original commands when + /// the user has configured a finite expanded-tree limit. + if (settings[Setting::max_expanded_ast_elements] && settings.isChanged("max_expanded_ast_elements")) + return commands; + + MutationCommands result; + result.reserve(commands.size()); + + for (size_t i = 0; i < commands.size();) + { + const auto first = getSingletonRowMaskUpdate(commands[i], max_combined_key_bytes); + if (!first || !hasStableRowMaskKeyTypes(*first, metadata_snapshot, source_columns) + || (commands[i].max_parser_depth && commands[i].max_parser_depth < DBMS_DEFAULT_MAX_PARSER_DEPTH) + || (commands[i].max_parser_backtracks && commands[i].max_parser_backtracks < DBMS_DEFAULT_MAX_PARSER_BACKTRACKS)) + { + result.push_back(commands[i++]); + continue; + } + + Tuple keys{first->key_value}; + UInt64 combined_key_bytes = first->key_value.size(); + size_t original_predicate_ast_size = first->predicate_ast_size; + std::vector prefixes{first->prefix_value}; + const bool can_mix_prefixes = first->key_from_uint128_array_join; + bool mixed_prefixes = false; + size_t end = i + 1; + while (end < commands.size() && keys.size() < max_keys_per_command) + { + const auto next = getSingletonRowMaskUpdate(commands[end], max_combined_key_bytes); + if (!next || next->key_value.size() > max_combined_key_bytes - combined_key_bytes + || commands[end].mutation_version != commands[i].mutation_version + || commands[end].max_parser_depth != commands[i].max_parser_depth + || commands[end].max_parser_backtracks != commands[i].max_parser_backtracks + || next->prefix_column != first->prefix_column + || (next->prefix_value != first->prefix_value + && (!can_mix_prefixes || !hasStableRowMaskKeyTypes(*next, metadata_snapshot, source_columns))) + || next->key_column != first->key_column + || next->key_from_uint128_array_join != first->key_from_uint128_array_join) + break; + + keys.emplace_back(next->key_value); + combined_key_bytes += next->key_value.size(); + original_predicate_ast_size += next->predicate_ast_size; + prefixes.emplace_back(next->prefix_value); + mixed_prefixes |= next->prefix_value != first->prefix_value; + ++end; + } + + if (keys.size() == 1) + { + result.push_back(commands[i]); + } + else + { + auto combined = commands[i]; + auto alter = combined.mutateAst(); + auto * predicate = alter->predicate->as(); + auto * membership = predicate->arguments->children[1]->as(); + if (mixed_prefixes) + { + auto pairs = makeASTFunction("tuple"); + for (size_t j = 0; j < keys.size(); ++j) + { + pairs->arguments->children.push_back(makeASTFunction( + "tuple", + make_intrusive(prefixes[j]), + makeASTFunction("toUInt128", make_intrusive(keys[j])))); + } + + static_cast(*alter).replace(alter->predicate, makeASTFunction( + "in", + makeASTFunction( + "tuple", + make_intrusive(first->prefix_column), + make_intrusive(first->key_column)), + pairs)); + } + else if (first->key_from_uint128_array_join) + { + auto * subquery = membership->arguments->children[1]->as(); + auto * union_query = subquery->children[0]->as(); + auto * selects = union_query->list_of_selects->as(); + auto * select = selects->children[0]->as(); + auto * projections = select->select()->as(); + auto * conversion = projections->children[0]->as(); + auto * array_join = conversion->arguments->children[0]->as(); + Array values; + values.reserve(keys.size()); + for (const auto & key : keys) + values.emplace_back(key); + array_join->arguments->children[0] = make_intrusive(std::move(values)); + } + else + { + membership->arguments->children[1] = make_intrusive(std::move(keys)); + } + alter.commit(); + const auto combined_ast = combined.ast(); + const auto combined_ast_size = combined_ast->size(); + if (combined_ast->predicate->size() > original_predicate_ast_size + || (settings[Setting::max_ast_elements] && combined_ast_size > settings[Setting::max_ast_elements]) + || (settings[Setting::max_expanded_ast_elements] + && combined_ast_size > settings[Setting::max_expanded_ast_elements])) + { + /// Keep the original commands if their combined predicate grows + /// the affected-row query or exceeds effective background limits. + for (size_t j = i; j < end; ++j) + result.push_back(commands[j]); + } + else + { + result.push_back(std::move(combined)); + } + } + i = end; + } + + return result; +} + /// Placeholder substream that `getColumnsForNewDataPart` records for a column that will be written /// later by the mutation and is therefore not yet present in the part. It is not a real stream and /// must never be resolved against the source part's checksums (a part may happen to contain a real @@ -3836,11 +4153,57 @@ bool MutateTask::prepare() }); } + /// Coalesce before splitting so a `DROP` or `RENAME` that the split omits + /// remains a grouping barrier. Keep file-rename planning based on the + /// original commands, including one `UPDATE` entry per original command. + const auto merge_tree_settings = ctx->data->getSettings(); + const UInt64 max_coalescing_commands = (*merge_tree_settings)[MergeTreeSetting::max_row_mask_update_coalescing_commands]; + const UInt64 max_coalescing_ast_bytes = (*merge_tree_settings)[MergeTreeSetting::max_row_mask_update_coalescing_ast_bytes]; + const UInt64 max_coalescing_keys = (*merge_tree_settings)[MergeTreeSetting::max_row_mask_update_coalescing_keys]; + const UInt64 max_coalescing_key_bytes = (*merge_tree_settings)[MergeTreeSetting::max_row_mask_update_coalescing_key_bytes]; + bool can_coalesce_row_mask_updates = (*merge_tree_settings)[MergeTreeSetting::enable_row_mask_update_coalescing] + && max_coalescing_commands && max_coalescing_ast_bytes && max_coalescing_keys > 1 && max_coalescing_key_bytes + && ctx->commands_for_part.size() > 1 + && ctx->commands_for_part.size() <= max_coalescing_commands + && isWidePart(ctx->source_part) + && isFullPartStorage(ctx->source_part->getDataPartStorage()) + && !MutationHelpers::hasDynamicColumnsWithoutRecordedSubstreams(ctx->source_part) + && !MutationHelpers::haveMutationsOfDynamicColumns(ctx->source_part, ctx->commands_for_part) + && !context_for_reading->getSettingsRef()[Setting::max_bytes_in_set]; + if (can_coalesce_row_mask_updates) + { + /// Copying the command list for part-local execution must stay bounded. + UInt64 input_bytes = 0; + for (const auto & command : ctx->commands_for_part) + { + if (command.ast_text.size() > max_coalescing_ast_bytes - input_bytes) + { + can_coalesce_row_mask_updates = false; + break; + } + input_bytes += command.ast_text.size(); + } + } + MutationCommands coalesced_commands; + const MutationCommands * commands_for_execution = &ctx->commands_for_part; + if (can_coalesce_row_mask_updates) + { + coalesced_commands = MutationHelpers::coalesceSingletonRowMaskUpdates( + ctx->commands_for_part, context_for_reading, ctx->metadata_snapshot, + ColumnsDescription(ctx->source_part->getColumns()), max_coalescing_keys, max_coalescing_key_bytes); + if (coalesced_commands.size() != ctx->commands_for_part.size()) + { + commands_for_execution = &coalesced_commands; + LOG_TRACE(ctx->log, "Coalesced row-mask updates for part {} (mutation commands {} -> {})", + ctx->source_part->name, ctx->commands_for_part.size(), coalesced_commands.size()); + } + } + auto is_storage_touched = isStorageTouchedByMutations( ctx->source_part, mutations_snapshot, ctx->metadata_snapshot, - ctx->commands_for_part, + *commands_for_execution, context_for_reading, [&my_ctx = *ctx](const Progress &) { my_ctx.checkOperationIsNotCanceled(); } ); @@ -3944,6 +4307,20 @@ bool MutateTask::prepare() suitable_for_ttl_optimization, ctx->log); + if (commands_for_execution != &ctx->commands_for_part) + { + ctx->for_interpreter.clear(); + MutationCommands unused_file_renames; + MutationHelpers::splitAndModifyMutationCommands( + ctx->source_part, + ctx->metadata_snapshot, + alter_conversions, + *commands_for_execution, + ctx->for_interpreter, + unused_file_renames, + ctx->log); + } + ctx->stage_progress = std::make_unique(1.0); bool lightweight_delete_mode = false; diff --git a/tests/integration/test_mutation_row_mask_set_limit/__init__.py b/tests/integration/test_mutation_row_mask_set_limit/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_mutation_row_mask_set_limit/configs/background_profile.xml b/tests/integration/test_mutation_row_mask_set_limit/configs/background_profile.xml new file mode 100644 index 0000000000000..dedb276986efd --- /dev/null +++ b/tests/integration/test_mutation_row_mask_set_limit/configs/background_profile.xml @@ -0,0 +1,3 @@ + + row_mask_limited + diff --git a/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_profile.xml b/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_profile.xml new file mode 100644 index 0000000000000..8bbb520e1488d --- /dev/null +++ b/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_profile.xml @@ -0,0 +1,3 @@ + + row_mask_expanded_ast_limited + diff --git a/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_users.xml b/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_users.xml new file mode 100644 index 0000000000000..e31ffa296a20f --- /dev/null +++ b/tests/integration/test_mutation_row_mask_set_limit/configs/expanded_ast_users.xml @@ -0,0 +1,7 @@ + + + + 100 + + + diff --git a/tests/integration/test_mutation_row_mask_set_limit/configs/limited_profile.xml b/tests/integration/test_mutation_row_mask_set_limit/configs/limited_profile.xml new file mode 100644 index 0000000000000..a372c95c45018 --- /dev/null +++ b/tests/integration/test_mutation_row_mask_set_limit/configs/limited_profile.xml @@ -0,0 +1,8 @@ + + + + 1 + throw + + + diff --git a/tests/integration/test_mutation_row_mask_set_limit/test.py b/tests/integration/test_mutation_row_mask_set_limit/test.py new file mode 100644 index 0000000000000..6c86615747273 --- /dev/null +++ b/tests/integration/test_mutation_row_mask_set_limit/test.py @@ -0,0 +1,358 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/background_profile.xml"], + user_configs=["configs/limited_profile.xml"], +) +replicated_node = cluster.add_instance("replicated_node", with_zookeeper=True) +expanded_ast_node = cluster.add_instance( + "expanded_ast_node", + main_configs=["configs/expanded_ast_profile.xml"], + user_configs=["configs/expanded_ast_users.xml"], +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_singleton_row_mask_updates_respect_background_set_limit(): + node.query( + "CREATE TABLE t (n UInt32, project_id UInt32, id String) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0" + ) + node.query("INSERT INTO t VALUES (1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c')") + + # Both singleton sets fit the background limit; a combined set would not. + node.query( + "ALTER TABLE t " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('b') " + "SETTINGS mutations_sync = 2" + ) + + assert node.query("SELECT n, id FROM t ORDER BY n") == "3\tc\n" + + +def test_mixed_project_subquery_updates_respect_background_set_limit(): + node.query( + "CREATE TABLE t_subquery (n UInt32, project_id UInt32, run_id UInt128) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0" + ) + node.query( + "INSERT INTO t_subquery VALUES " + "(1, 1, 1), (2, 2, 2), (3, 1, 2), (4, 2, 1)" + ) + + # Each one-row set fits. A tuple set with two project/run pairs does not. + node.query( + "ALTER TABLE t_subquery " + "UPDATE _row_exists = 0 WHERE (project_id = 1) " + "AND (run_id IN (SELECT toUInt128(arrayJoin(['1'])))), " + "UPDATE _row_exists = 0 WHERE (project_id = 2) " + "AND (run_id IN (SELECT toUInt128(arrayJoin(['2'])))) " + "SETTINGS mutations_sync = 2" + ) + + assert node.query("SELECT n, project_id, run_id FROM t_subquery ORDER BY n") == "3\t1\t2\n4\t2\t1\n" + + +def test_mixed_project_updates_respect_background_expanded_ast_limit(): + expanded_ast_node.query( + "CREATE TABLE t_expanded_ast (n UInt32, project_id UInt32, run_id UInt128) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0" + ) + expanded_ast_node.query( + "INSERT INTO t_expanded_ast SELECT number + 1, number + 1, number + 1 " + "FROM numbers(11)" + ) + expanded_ast_node.query( + "ALTER TABLE t_expanded_ast ADD INDEX idx project_id TYPE minmax GRANULARITY 1" + ) + assert expanded_ast_node.query( + "SELECT part_type, part_storage_type FROM system.parts " + "WHERE database = currentDatabase() AND table = 't_expanded_ast' AND active" + ) == "Wide\tFull\n" + + commands = [ + "UPDATE _row_exists = 0 WHERE project_id = " + f"{n} AND run_id IN (SELECT toUInt128(arrayJoin(['{n}'])))" + for n in range(1, 11) + ] + commands.append("MATERIALIZE INDEX idx") + expanded_ast_node.query( + "ALTER TABLE t_expanded_ast " + + ", ".join(commands) + + " SETTINGS mutations_sync = 2, allow_nondeterministic_mutations = 1" + ) + assert expanded_ast_node.query("SELECT n FROM t_expanded_ast ORDER BY n") == "11\n" + assert expanded_ast_node.query( + "SELECT count(), countIf(is_done), countIf(latest_fail_reason != '') " + "FROM system.mutations WHERE database = currentDatabase() " + "AND table = 't_expanded_ast'" + ) == "11\t11\t0\n" + assert not expanded_ast_node.contains_in_log("Coalesced row-mask updates for part") + + +def test_separate_replicated_mutations_coalesce_during_part_rewrite(): + replicated_node.query( + "CREATE TABLE t_replicated " + "(n UInt64, project_id UInt32, run_id UInt128, payload UInt8) " + "ENGINE = ReplicatedMergeTree(" + "'/clickhouse/tables/test_mutation_row_mask_set_limit/t_replicated', 'r1') " + "ORDER BY n SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0, " + "number_of_free_entries_in_pool_to_execute_mutation = 0, " + "number_of_free_entries_in_pool_to_execute_optimize_entire_partition = 0" + ) + replicated_node.query( + "INSERT INTO t_replicated VALUES " + "(1, 1, 1, 0), (2, 1, 2, 0), (3, 1, 3, 0), " + "(4, 2, 1, 0), (5, 2, 2, 0), (6, 2, 3, 0), (7, 3, 1, 0)" + ) + replicated_node.query("SYSTEM STOP MERGES t_replicated") + for project, key in [(1, 1), (2, 1), (1, 2)]: + replicated_node.query( + "ALTER TABLE t_replicated UPDATE _row_exists = 0 " + f"WHERE project_id = {project} AND " + f"run_id IN (SELECT toUInt128(arrayJoin(['{key}']))) " + "SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1" + ) + replicated_node.query( + "ALTER TABLE t_replicated UPDATE payload = 9 WHERE n = 6 " + "SETTINGS mutations_sync = 0" + ) + for project, key in [(1, 3), (2, 2)]: + replicated_node.query( + "ALTER TABLE t_replicated UPDATE _row_exists = 0 " + f"WHERE project_id = {project} AND " + f"run_id IN (SELECT toUInt128(arrayJoin(['{key}']))) " + "SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1" + ) + + assert replicated_node.query( + "SELECT count() FROM system.mutations " + "WHERE database = currentDatabase() AND table = 't_replicated' AND NOT is_done" + ) == "6\n" + + replicated_node.query("SYSTEM START MERGES t_replicated") + replicated_node.query( + "ALTER TABLE t_replicated UPDATE payload = payload WHERE n = 7 " + "SETTINGS mutations_sync = 2" + ) + assert replicated_node.query( + "SELECT n, project_id, run_id, payload FROM t_replicated ORDER BY n" + ) == "6\t2\t3\t9\n7\t3\t1\t0\n" + assert replicated_node.query( + "SELECT count(), countIf(is_done) FROM system.mutations " + "WHERE database = currentDatabase() AND table = 't_replicated'" + ) == "7\t7\n" + assert replicated_node.contains_in_log("Coalesced row-mask updates for part") + + +def test_row_mask_coalescing_table_settings(): + replicated_node.query( + "CREATE TABLE t_coalescing_settings " + "(n UInt32, project_id UInt32, id String) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0, " + "enable_row_mask_update_coalescing = 0" + ) + replicated_node.query( + "INSERT INTO t_coalescing_settings VALUES " + "(1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 1, 'd'), " + "(5, 1, 'e'), (6, 1, 'f'), (7, 1, 'g'), (8, 1, 'h'), " + "(9, 1, 'i'), (10, 1, 'j'), (11, 1, 'k'), (12, 1, 'l'), (13, 1, 'm')" + ) + + def delete_pair(first, second): + replicated_node.query( + "ALTER TABLE t_coalescing_settings " + f"UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('{first}'), " + f"UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('{second}') " + "SETTINGS mutations_sync = 2" + ) + + coalesced_count = int(replicated_node.count_in_log("Coalesced row-mask updates for part")) + delete_pair("a", "b") + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + replicated_node.query( + "ALTER TABLE t_coalescing_settings MODIFY SETTING enable_row_mask_update_coalescing = 1" + ) + delete_pair("c", "d") + coalesced_count += 1 + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + replicated_node.query( + "ALTER TABLE t_coalescing_settings MODIFY SETTING max_row_mask_update_coalescing_keys = 1" + ) + delete_pair("e", "f") + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + replicated_node.query( + "ALTER TABLE t_coalescing_settings MODIFY SETTING " + "max_row_mask_update_coalescing_keys = 2, max_row_mask_update_coalescing_key_bytes = 1" + ) + delete_pair("g", "h") + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + replicated_node.query( + "ALTER TABLE t_coalescing_settings MODIFY SETTING " + "max_row_mask_update_coalescing_key_bytes = 2, max_row_mask_update_coalescing_commands = 1" + ) + delete_pair("i", "j") + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + replicated_node.query( + "ALTER TABLE t_coalescing_settings MODIFY SETTING " + "max_row_mask_update_coalescing_commands = 2, max_row_mask_update_coalescing_ast_bytes = 1" + ) + delete_pair("k", "l") + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + assert replicated_node.query("SELECT n, id FROM t_coalescing_settings ORDER BY n") == "13\tm\n" + + +def test_nested_alias_is_not_coalesced(): + coalesced_count = int(replicated_node.count_in_log("Coalesced row-mask updates for part")) + for enabled in (0, 1): + table = f"t_alias_coalescing_{enabled}" + replicated_node.query( + f"CREATE TABLE {table} (n UInt32, project_id UInt32, id String) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0, " + f"enable_row_mask_update_coalescing = {enabled}" + ) + replicated_node.query( + f"INSERT INTO {table} VALUES " + "(1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 2, 'd')" + ) + assert replicated_node.query( + "SELECT part_type, part_storage_type FROM system.parts " + f"WHERE database = currentDatabase() AND table = '{table}' AND active" + ) == "Wide\tFull\n" + assert replicated_node.query( + f"SELECT n FROM {table} WHERE project_id = 1 " + "AND id IN (('a' AS id)) ORDER BY n" + ) == "1\n2\n3\n" + replicated_node.query( + f"ALTER TABLE {table} " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN (('a' AS id)), " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN (('a' AS id)) " + "SETTINGS mutations_sync = 2" + ) + assert replicated_node.query(f"SELECT n, id FROM {table} ORDER BY n") == "4\td\n" + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + +def test_assignment_alias_is_not_coalesced(): + coalesced_count = int(replicated_node.count_in_log("Coalesced row-mask updates for part")) + for enabled in (0, 1): + table = f"t_assignment_alias_coalescing_{enabled}" + replicated_node.query( + f"CREATE TABLE {table} (n UInt32, project_id UInt32, id String) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0, " + f"enable_row_mask_update_coalescing = {enabled}" + ) + replicated_node.query( + f"INSERT INTO {table} VALUES " + "(1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 2, 'd')" + ) + assert replicated_node.query( + "SELECT part_type, part_storage_type FROM system.parts " + f"WHERE database = currentDatabase() AND table = '{table}' AND active" + ) == "Wide\tFull\n" + replicated_node.query( + f"ALTER TABLE {table} " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), " + "UPDATE _row_exists = (0 AS project_id) WHERE project_id = 1 AND id IN ('b') " + "SETTINGS mutations_sync = 2, prefer_column_name_to_alias = 0" + ) + assert replicated_node.query(f"SELECT n, id FROM {table} ORDER BY n") == "2\tb\n3\tc\n4\td\n" + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + + +def test_same_project_uint128_subqueries_coalesce(): + replicated_node.query( + "CREATE TABLE t_same_project_subqueries (n UInt32, project_id UInt32, run_id UInt128) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0" + ) + replicated_node.query( + "INSERT INTO t_same_project_subqueries VALUES (1, 1, 1), (2, 1, 2), (3, 1, 3)" + ) + assert replicated_node.query( + "SELECT part_type, part_storage_type FROM system.parts " + "WHERE database = currentDatabase() AND table = 't_same_project_subqueries' AND active" + ) == "Wide\tFull\n" + + coalesced_count = int(replicated_node.count_in_log("Coalesced row-mask updates for part")) + replicated_node.query( + "ALTER TABLE t_same_project_subqueries " + "UPDATE _row_exists = 0 WHERE project_id = 1 " + "AND run_id IN (SELECT toUInt128(arrayJoin(['1']))), " + "UPDATE _row_exists = 0 WHERE project_id = 1 " + "AND run_id IN (SELECT toUInt128(arrayJoin(['2']))) " + "SETTINGS mutations_sync = 2, allow_nondeterministic_mutations = 1" + ) + assert replicated_node.query( + "SELECT n, run_id FROM t_same_project_subqueries ORDER BY n" + ) == "3\t3\n" + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count + 1 + + +def test_rollup_subquery_is_not_coalesced(): + coalesced_count = int(replicated_node.count_in_log("Coalesced row-mask updates for part")) + for enabled in (0, 1): + table = f"t_rollup_coalescing_{enabled}" + replicated_node.query( + f"CREATE TABLE {table} (n UInt32, project_id UInt32, run_id UInt128) " + "ENGINE = MergeTree ORDER BY n " + "SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, " + "min_bytes_for_full_part_storage = 0, " + f"enable_row_mask_update_coalescing = {enabled}" + ) + replicated_node.query( + f"INSERT INTO {table} VALUES (1, 1, 0), (2, 1, 1), (3, 2, 2)" + ) + assert replicated_node.query( + "SELECT part_type, part_storage_type FROM system.parts " + f"WHERE database = currentDatabase() AND table = '{table}' AND active" + ) == "Wide\tFull\n" + assert replicated_node.query( + f"SELECT count() FROM {table} WHERE project_id = 1 " + "AND run_id IN (SELECT toUInt128(arrayJoin(['1'])) GROUP BY ALL WITH ROLLUP) " + "SETTINGS group_by_use_nulls = 0" + ) == "2\n" + replicated_node.query( + f"ALTER TABLE {table} " + "UPDATE _row_exists = 0 WHERE project_id = 1 AND run_id IN " + "(SELECT toUInt128(arrayJoin(['1'])) GROUP BY ALL WITH ROLLUP), " + "UPDATE _row_exists = 0 WHERE project_id = 2 AND run_id IN " + "(SELECT toUInt128(arrayJoin(['2']))) " + "SETTINGS mutations_sync = 2, allow_nondeterministic_mutations = 1, " + "group_by_use_nulls = 0" + ) + assert replicated_node.query(f"SELECT n FROM {table} ORDER BY n") == "" + assert int(replicated_node.count_in_log("Coalesced row-mask updates for part")) == coalesced_count diff --git a/tests/queries/0_stateless/05256_coalesce_row_mask_updates.reference b/tests/queries/0_stateless/05256_coalesce_row_mask_updates.reference new file mode 100644 index 0000000000000..d1b012f02592a --- /dev/null +++ b/tests/queries/0_stateless/05256_coalesce_row_mask_updates.reference @@ -0,0 +1,12 @@ +wide 2 1 c 1 +wide 7 1 f 0 +compact 2 1 c 1 +compact 7 1 f 0 +again 2 c +again 7 f +alias_off 2 b +alias_off 3 c +alias_off 4 d +alias_on 2 b +alias_on 3 c +alias_on 4 d diff --git a/tests/queries/0_stateless/05256_coalesce_row_mask_updates.sql b/tests/queries/0_stateless/05256_coalesce_row_mask_updates.sql new file mode 100644 index 0000000000000..96858cb72631c --- /dev/null +++ b/tests/queries/0_stateless/05256_coalesce_row_mask_updates.sql @@ -0,0 +1,84 @@ +DROP TABLE IF EXISTS test_05256_wide; +DROP TABLE IF EXISTS test_05256_compact; +DROP TABLE IF EXISTS test_05256_assignment_alias_off; +DROP TABLE IF EXISTS test_05256_assignment_alias_on; + +CREATE TABLE test_05256_wide (n UInt64, project_id UInt32, id String, payload UInt8) +ENGINE = MergeTree ORDER BY n +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0; + +CREATE TABLE test_05256_compact (n UInt64, project_id UInt32, id String, payload UInt8) +ENGINE = MergeTree ORDER BY n +SETTINGS min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000; + +INSERT INTO test_05256_wide VALUES + (0, 1, 'a', 0), (1, 1, 'b', 0), (2, 1, 'c', 0), (3, 2, 'a', 0), + (4, 2, 'd', 0), (5, 1, 'd', 0), (6, 1, 'e', 0), (7, 1, 'f', 0); +INSERT INTO test_05256_compact SELECT * FROM test_05256_wide; + +ALTER TABLE test_05256_wide + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('b'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('b'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('missing'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN (concat('no', 'match')), + UPDATE payload = 1 WHERE project_id = 1 AND id IN ('c'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('d'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('e'), + UPDATE _row_exists = 0 WHERE project_id = 2 AND id IN ('a'), + UPDATE _row_exists = 0 WHERE project_id = 2 AND id IN ('d') +SETTINGS mutations_sync = 2; + +ALTER TABLE test_05256_compact + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('b'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('b'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('missing'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN (concat('no', 'match')), + UPDATE payload = 1 WHERE project_id = 1 AND id IN ('c'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('d'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('e'), + UPDATE _row_exists = 0 WHERE project_id = 2 AND id IN ('a'), + UPDATE _row_exists = 0 WHERE project_id = 2 AND id IN ('d') +SETTINGS mutations_sync = 2; + +SELECT 'wide', n, project_id, id, payload FROM test_05256_wide ORDER BY n; +SELECT 'compact', n, project_id, id, payload FROM test_05256_compact ORDER BY n; + +ALTER TABLE test_05256_wide + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('absent') +SETTINGS mutations_sync = 2; +SELECT 'again', n, id FROM test_05256_wide ORDER BY n; + +CREATE TABLE test_05256_assignment_alias_off (n UInt32, project_id UInt32, id String) +ENGINE = MergeTree ORDER BY n +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, enable_row_mask_update_coalescing = 0; + +CREATE TABLE test_05256_assignment_alias_on (n UInt32, project_id UInt32, id String) +ENGINE = MergeTree ORDER BY n +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, enable_row_mask_update_coalescing = 1; + +INSERT INTO test_05256_assignment_alias_off VALUES + (1, 1, 'a'), (2, 1, 'b'), (3, 1, 'c'), (4, 2, 'd'); +INSERT INTO test_05256_assignment_alias_on SELECT * FROM test_05256_assignment_alias_off; + +ALTER TABLE test_05256_assignment_alias_off + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), + UPDATE _row_exists = (0 AS project_id) WHERE project_id = 1 AND id IN ('b') +SETTINGS mutations_sync = 2, prefer_column_name_to_alias = 0; +ALTER TABLE test_05256_assignment_alias_on + UPDATE _row_exists = 0 WHERE project_id = 1 AND id IN ('a'), + UPDATE _row_exists = (0 AS project_id) WHERE project_id = 1 AND id IN ('b') +SETTINGS mutations_sync = 2, prefer_column_name_to_alias = 0; + +SELECT 'alias_off', n, id FROM test_05256_assignment_alias_off ORDER BY n; +SELECT 'alias_on', n, id FROM test_05256_assignment_alias_on ORDER BY n; + +DROP TABLE test_05256_wide; +DROP TABLE test_05256_compact; +DROP TABLE test_05256_assignment_alias_off; +DROP TABLE test_05256_assignment_alias_on; diff --git a/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.reference b/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.reference new file mode 100644 index 0000000000000..87bee9ba85bb8 --- /dev/null +++ b/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.reference @@ -0,0 +1,3 @@ +3 1 3 1 +7 2 5 0 +8 3 1 0 diff --git a/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.sql b/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.sql new file mode 100644 index 0000000000000..3d79c159b5533 --- /dev/null +++ b/tests/queries/0_stateless/05257_coalesce_row_mask_subquery.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test_05257; + +CREATE TABLE test_05257 +( + n UInt64, + project_id UInt32, + field_hash UInt32, + run_id UInt128, + payload UInt8 +) +ENGINE = AggregatingMergeTree +ORDER BY (project_id, field_hash, run_id) +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0; + +INSERT INTO test_05257 VALUES + (1, 1, 1, 1, 0), (2, 1, 2, 2, 0), (3, 1, 3, 3, 0), + (4, 2, 4, 1, 0), (5, 1, 5, 4, 0), (6, 1, 6, 5, 0), + (7, 2, 7, 5, 0), (8, 3, 8, 1, 0); + +ALTER TABLE test_05257 + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['1'])))), + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['2'])))), + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['2'])))), + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['99'])))), + UPDATE payload = 1 WHERE project_id = 1 AND run_id = 3, + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['4'])))), + UPDATE _row_exists = 0 WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['5'])))), + UPDATE _row_exists = 0 WHERE (project_id = 2) AND (run_id IN (SELECT toUInt128(arrayJoin(['1'])))) +SETTINGS mutations_sync = 2; + +SELECT n, project_id, run_id, payload FROM test_05257 ORDER BY n; + +DROP TABLE test_05257; diff --git a/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.reference b/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.reference new file mode 100644 index 0000000000000..2dd35fa672642 --- /dev/null +++ b/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.reference @@ -0,0 +1,3 @@ +6 2 3 9 +7 3 1 0 +7 7 diff --git a/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.sql b/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.sql new file mode 100644 index 0000000000000..550305ec6d027 --- /dev/null +++ b/tests/queries/0_stateless/05258_coalesce_row_mask_separate_mutations.sql @@ -0,0 +1,51 @@ +DROP TABLE IF EXISTS test_05258; + +CREATE TABLE test_05258 +( + n UInt64, + project_id UInt32, + field_hash UInt32, + run_id UInt128, + payload UInt8 +) +ENGINE = AggregatingMergeTree +ORDER BY (project_id, field_hash, run_id) +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + number_of_free_entries_in_pool_to_execute_mutation = 0, + number_of_free_entries_in_pool_to_execute_optimize_entire_partition = 0; + +INSERT INTO test_05258 VALUES + (1, 1, 1, 1, 0), (2, 1, 2, 2, 0), (3, 1, 3, 3, 0), + (4, 2, 4, 1, 0), (5, 2, 5, 2, 0), (6, 2, 6, 3, 0), + (7, 3, 7, 1, 0); + +SYSTEM STOP MERGES test_05258; + +ALTER TABLE test_05258 UPDATE _row_exists = 0 +WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['1'])))) +SETTINGS mutations_sync = 0; +ALTER TABLE test_05258 UPDATE _row_exists = 0 +WHERE (project_id = 2) AND (run_id IN (SELECT toUInt128(arrayJoin(['1'])))) +SETTINGS mutations_sync = 0; +ALTER TABLE test_05258 UPDATE _row_exists = 0 +WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['2'])))) +SETTINGS mutations_sync = 0; + +ALTER TABLE test_05258 UPDATE payload = 9 WHERE n = 6 SETTINGS mutations_sync = 0; + +ALTER TABLE test_05258 UPDATE _row_exists = 0 +WHERE (project_id = 1) AND (run_id IN (SELECT toUInt128(arrayJoin(['3'])))) +SETTINGS mutations_sync = 0; +ALTER TABLE test_05258 UPDATE _row_exists = 0 +WHERE (project_id = 2) AND (run_id IN (SELECT toUInt128(arrayJoin(['2'])))) +SETTINGS mutations_sync = 0; + +SYSTEM START MERGES test_05258; +ALTER TABLE test_05258 UPDATE payload = payload WHERE n = 7 SETTINGS mutations_sync = 2; + +SELECT n, project_id, run_id, payload FROM test_05258 ORDER BY n; +SELECT count(), countIf(is_done) FROM system.mutations +WHERE database = currentDatabase() AND table = 'test_05258'; + +DROP TABLE test_05258; From 01b983a8e70a62904db316de2a096369396aec1b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 12:07:25 +0000 Subject: [PATCH 02/81] Backport #120355 to 26.8: Reduce Lock Contention on Shared Context Mutex --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 67 +++++++++++++++++--------- src/Common/ZooKeeper/ZooKeeperImpl.h | 4 ++ src/Interpreters/Context.cpp | 25 +++++++--- src/Interpreters/Context.h | 1 + 4 files changed, 67 insertions(+), 30 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 039ff73ba69c7..ff92d0e1963e5 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -2297,41 +2297,64 @@ int64_t ZooKeeper::getConnectionXid() const } -std::shared_ptr ZooKeeper::getZooKeeperLog() +bool ZooKeeper::resolveSystemLogs() { - if (auto maybe_zk_log = std::atomic_load_explicit(&zk_log, std::memory_order_relaxed)) + while (true) { - return maybe_zk_log; + auto state = system_logs_state.load(); + if (state == SystemLogsState::Resolved) + return true; + if (state == SystemLogsState::Unresolved && system_logs_state.compare_exchange_strong(state, SystemLogsState::InProgress)) + break; + system_logs_state.wait(SystemLogsState::InProgress); } - if (const auto maybe_global_context = Context::getGlobalContextInstance()) + auto set_state = [&](SystemLogsState state) { - if (auto maybe_zk_log = maybe_global_context->getZooKeeperLog()) + system_logs_state = state; + system_logs_state.notify_all(); + }; + + try + { + if (const auto global_context = Context::getGlobalContextInstance()) { - std::atomic_store_explicit(&zk_log, maybe_zk_log, std::memory_order_relaxed); - return maybe_zk_log; + if (!global_context->hasSystemLogs()) + { + set_state(SystemLogsState::Unresolved); + return false; + } + + if (!zk_log) + zk_log = global_context->getZooKeeperLog(); + if (!aggregated_zookeeper_log) + aggregated_zookeeper_log = global_context->getAggregatedZooKeeperLog(); } } + catch (...) + { + set_state(SystemLogsState::Unresolved); + throw; + } - return nullptr; + set_state(SystemLogsState::Resolved); + return true; } -std::shared_ptr ZooKeeper::getAggregatedZooKeeperLog() + +std::shared_ptr ZooKeeper::getZooKeeperLog() { - if (auto maybe_aggregated_zookeeper_log = std::atomic_load_explicit(&aggregated_zookeeper_log, std::memory_order_relaxed)) - { - return maybe_aggregated_zookeeper_log; - } + if (!resolveSystemLogs()) + return nullptr; - if (const auto maybe_global_context = Context::getGlobalContextInstance()) - { - if (auto maybe_aggregated_zookeeper_log = maybe_global_context->getAggregatedZooKeeperLog()) - { - std::atomic_store_explicit(&aggregated_zookeeper_log, maybe_aggregated_zookeeper_log, std::memory_order_relaxed); - return maybe_aggregated_zookeeper_log; - } - } + return zk_log; +} + +std::shared_ptr ZooKeeper::getAggregatedZooKeeperLog() +{ + if (!resolveSystemLogs()) + return nullptr; - return nullptr; + return aggregated_zookeeper_log; } #ifdef ZOOKEEPER_LOG diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 8a8461982bc5d..3e4d6e4365951 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -414,6 +414,10 @@ class ZooKeeper final : public IKeeper std::shared_ptr zk_log; std::shared_ptr aggregated_zookeeper_log; + bool resolveSystemLogs(); + enum class SystemLogsState { Unresolved, InProgress, Resolved }; + std::atomic system_logs_state{SystemLogsState::Unresolved}; + std::atomic last_zxid_seen; /// Timestamp of the last data received from the server (any kind: response, diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 54cc24c6f638f..8626af9c9793d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -704,7 +704,8 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr ddl_worker TSA_GUARDED_BY(mutex); /// Process ddl commands from zk. LoadTaskPtr ddl_worker_startup_task; /// To postpone `ddl_worker->startup()` after all tables startup /// Rules for selecting the compression settings, depending on the size of the part. - mutable std::unique_ptr compression_codec_selector TSA_GUARDED_BY(mutex); + mutable OnceFlag compression_codec_selector_initialized; + mutable std::unique_ptr compression_codec_selector; /// Storage disk chooser for MergeTree engines mutable std::shared_ptr merge_tree_disk_selector TSA_GUARDED_BY(storage_policies_mutex); /// Storage policy chooser for MergeTree engines @@ -7046,6 +7047,16 @@ std::shared_ptr Context::getSessionLog() const } +bool Context::hasSystemLogs() const +{ + std::lock_guard lock(mutex_shared_context); + if (!shared) + return false; + + SharedLockGuard lock2(shared->mutex); + return shared->system_logs != nullptr; +} + std::shared_ptr Context::getZooKeeperLog() const { std::lock_guard lock(mutex_shared_context); @@ -7284,18 +7295,16 @@ void Context::setDashboardsConfig(const Poco::Util::AbstractConfiguration & conf CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { - std::lock_guard lock(shared->mutex); - - if (!shared->compression_codec_selector) + callOnce(shared->compression_codec_selector_initialized, [&] { constexpr auto config_name = "compression"; - const auto & config = shared->getConfigRefWithLock(lock); + auto config = shared->getConfig(); - if (config.has(config_name)) - shared->compression_codec_selector = std::make_unique(config, "compression"); + if (config->has(config_name)) + shared->compression_codec_selector = std::make_unique(*config, config_name); else shared->compression_codec_selector = std::make_unique(); - } + }); return shared->compression_codec_selector->choose(part_size, part_size_ratio); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 522eef1fa017b..a83c02bfbb18a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1715,6 +1715,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// Call after initialization before using system logs. Call for global context. void initializeSystemLogs(); + bool hasSystemLogs() const; /// Call after initialization before using trace collector. void createTraceCollector(); From bfccf1ae30c0b85ba17a5b56285795ffb4f1728a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 12:37:45 +0000 Subject: [PATCH 03/81] Backport #119510 to 26.8: Bump `mariadb-connector-c` to v3.4.10 --- contrib/mariadb-connector-c | 2 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 2 +- .../helpers/mysql_cert/generate_certs.sh | 7 +++- .../helpers/mysql_cert/server-cert.pem | 37 ++++++++++--------- .../helpers/mysql_cert/server-req.pem | 16 ++++---- 5 files changed, 36 insertions(+), 28 deletions(-) diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 111ec1a5958cf..b3c09ab333c2e 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 111ec1a5958cf984f50a28b0fb82a4087918d677 +Subproject commit b3c09ab333c2ef9277291e721871a2bafca6e8bf diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 576d02de4b5e3..1f0e9a8695b20 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -45,7 +45,7 @@ MATH(EXPR MARIADB_VERSION_ID "${MARIADB_CLIENT_VERSION_MAJOR} * 10000 + set(CPACK_PACKAGE_VERSION_MAJOR 3) set(CPACK_PACKAGE_VERSION_MINOR 4) -set(CPACK_PACKAGE_VERSION_PATCH 9) +set(CPACK_PACKAGE_VERSION_PATCH 10) set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") MATH(EXPR MARIADB_PACKAGE_VERSION_ID "${CPACK_PACKAGE_VERSION_MAJOR} * 10000 + ${CPACK_PACKAGE_VERSION_MINOR} * 100 + diff --git a/tests/integration/helpers/mysql_cert/generate_certs.sh b/tests/integration/helpers/mysql_cert/generate_certs.sh index 9ee4e9b5dbe78..ab1524b3fbe16 100755 --- a/tests/integration/helpers/mysql_cert/generate_certs.sh +++ b/tests/integration/helpers/mysql_cert/generate_certs.sh @@ -6,9 +6,12 @@ openssl req -new -x509 -nodes -days 3600 -key ca-key.pem -out ca.pem -subj "/C=A # Create server certificate, remove passphrase, and sign it # server-cert.pem = public key, server-key.pem = private key -openssl req -newkey rsa:2048 -days 3600 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=AU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" +# The subjectAltName must list every hostname clients connect to, because +# mariadb-connector-c (since 3.4.10) verifies the server hostname whenever +# a CA certificate is configured. +openssl req -newkey rsa:2048 -days 3600 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=AU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" -addext "subjectAltName=DNS:mysql80,DNS:mysql2,DNS:mysql3,DNS:mysql4" openssl rsa -in server-key.pem -out server-key.pem -openssl x509 -req -in server-req.pem -days 3600 -CA ca.pem -CAkey ca-key.pem -set_serial 01 -out server-cert.pem +openssl x509 -req -in server-req.pem -days 3600 -CA ca.pem -CAkey ca-key.pem -set_serial 01 -copy_extensions copy -out server-cert.pem # Create client certificate, remove passphrase, and sign it # client-cert.pem = public key, client-key.pem = private key diff --git a/tests/integration/helpers/mysql_cert/server-cert.pem b/tests/integration/helpers/mysql_cert/server-cert.pem index 1a70507762680..4ffaeeb8a2242 100644 --- a/tests/integration/helpers/mysql_cert/server-cert.pem +++ b/tests/integration/helpers/mysql_cert/server-cert.pem @@ -1,19 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDHDCCAgQCAQEwDQYJKoZIhvcNAQELBQAwUjELMAkGA1UEBhMCQVUxEzARBgNV -BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 -ZDELMAkGA1UEAwwCY2EwHhcNMjQxMjE3MjI1NzM3WhcNMzQxMDI2MjI1NzM3WjBW -MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8wDQYDVQQDDAZzZXJ2ZXIwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgEK5ITXEHhuVYHX8wJ80Mnnh5WBISj/g6 -HnCjrOTqwq4xtbSaRcsJlzkDWnNhJ010n9zaw99Ci5SU86zS1QHI2JfVmCJuDN28 -wkmCu6zEgMNgk3p0DHfnxhZEhFnnPTXEi0W/gseIMaE1y5QNQxoBMbcjMjFEdX2U -k/+hUYc6X2rn/Tn90BsUZ65BEp3XYuOIelTzGtwbyMLPgdVcsMf/nPsHbtE2qh7L -6cLEkiWMbCTT8EZoA1GrSc6uRcCbSon4RsiwJLhBHVq0XpXqVkHDllgrJjxVeHu8 -4LhTcZJKk2x+cbfp8DkfcbxG6KaHMbXpOcCHie2RTTFhK+0n5LfhAgMBAAEwDQYJ -KoZIhvcNAQELBQADggEBAAlIcTnVu5F3VQ2R15Q745lnzP3N5v8RgVJgelmzalqO -hI2K9fOb0h2aL98Muf1mk5kaSUboKzlg82Wgo/3yCuiPA/sesDMbseTnAPX/IDTi -f0IikyZBvnhLcYnOI23gUaRcOSYMC5k+JleQ+hHWNRqyXyPIhOy4A4e7H42UkIc+ -sExCxiN4Y9bbBTTUYaL4F5m3S4JeuwfXQ7ym2i9ytJqIAbt3GDztkCvYT5DuLkCk -SJiIwLfSsH2oN/mOqHIu1cKj7ynhdbEmR62tpT5XyNBoHgOuF8UoFfttsDTgKFgf -h4bTlddfoWXbhowpu7oiSpNkMAgsakxknRS/iJ1V5xc= +MIIDkTCCAnmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBSMQswCQYDVQQGEwJBVTET +MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ +dHkgTHRkMQswCQYDVQQDDAJjYTAeFw0yNjEwMTIxMTI0MDZaFw0zNjA4MjAxMTI0 +MDZaMFYxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQK +DBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBnNlcnZlcjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKAQrkhNcQeG5VgdfzAnzQyeeHlY +EhKP+DoecKOs5OrCrjG1tJpFywmXOQNac2EnTXSf3NrD30KLlJTzrNLVAcjYl9WY +Im4M3bzCSYK7rMSAw2CTenQMd+fGFkSEWec9NcSLRb+Cx4gxoTXLlA1DGgExtyMy +MUR1fZST/6FRhzpfauf9Of3QGxRnrkESnddi44h6VPMa3BvIws+B1Vywx/+c+wdu +0TaqHsvpwsSSJYxsJNPwRmgDUatJzq5FwJtKifhGyLAkuEEdWrRelepWQcOWWCsm +PFV4e7zguFNxkkqTbH5xt+nwOR9xvEbopocxtek5wIeJ7ZFNMWEr7Sfkt+ECAwEA +AaNuMGwwKgYDVR0RBCMwIYIHbXlzcWw4MIIGbXlzcWwyggZteXNxbDOCBm15c3Fs +NDAdBgNVHQ4EFgQUAKvOBDGHp597AgEdDZoLbmYd38cwHwYDVR0jBBgwFoAUTSGM +MheG8XGK4fbHcq9fpyFdD8AwDQYJKoZIhvcNAQELBQADggEBAIG5h1cDF0k8ZE3w +2w+SA5mrTzTA0HRMIMyJwBabar0vEkE/g5qSxQjAtb0U2EEarMri5qOiYwqkWai3 +uo2kR186MbEAvnz+BMdN0+Kl0hMqzxFpuhKvLt2Jlu9Vlmwrl2krXJzsB29sAiCt +qFa/OY/1+6MAtxbzmtAIK+6MFFZHKyxqO8nkuy1/LU18fRz9dtFIEwtqOrJPEzXi +hObmZ2+ssB3bvB2VNRVBV0y78NpqZkTej9CWBgzlny9VbavrEPjzFJNGPFab2Fdu +Kh/wv+URpMHhgSaJa3NG1yI/33so9eSkqXjTTqs8U+ESAvYMNJGgO+WuGOdNoSf+ +hdj17/c= -----END CERTIFICATE----- diff --git a/tests/integration/helpers/mysql_cert/server-req.pem b/tests/integration/helpers/mysql_cert/server-req.pem index ede50528d4734..52c67618d5c56 100644 --- a/tests/integration/helpers/mysql_cert/server-req.pem +++ b/tests/integration/helpers/mysql_cert/server-req.pem @@ -1,5 +1,5 @@ -----BEGIN CERTIFICATE REQUEST----- -MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx +MIIC2DCCAcACAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy dmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoBCuSE1xB4blWB1/ MCfNDJ54eVgSEo/4Oh5wo6zk6sKuMbW0mkXLCZc5A1pzYSdNdJ/c2sPfQouUlPOs @@ -7,10 +7,12 @@ MCfNDJ54eVgSEo/4Oh5wo6zk6sKuMbW0mkXLCZc5A1pzYSdNdJ/c2sPfQouUlPOs DUMaATG3IzIxRHV9lJP/oVGHOl9q5/05/dAbFGeuQRKd12LjiHpU8xrcG8jCz4HV XLDH/5z7B27RNqoey+nCxJIljGwk0/BGaANRq0nOrkXAm0qJ+EbIsCS4QR1atF6V 6lZBw5ZYKyY8VXh7vOC4U3GSSpNsfnG36fA5H3G8RuimhzG16TnAh4ntkU0xYSvt -J+S34QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAAVKb/FXWi/19JRFMUvFj3P8 -mfKn+OJ1ekpJZg6yTMFE0DVO3foXlMtpUN8URuOX92nCCB6JOT7X0c7rq6/g5G/+ -8YW8MGyJsWgNJf9yfkOzV+uMwkVyW3vln9Z9ApooBlryGyBhwQwiLcAt7DbGk3/u -QbHd2t+h6vzjYQZeSaX7y5kbSlybYYhUwWV2iwTyJSUgqsWuJM3xz5+yTnDXDvrJ -20I5hs7EJYM1bcOYFKHzgRmfXUojeT6fcEyoSF5zblbELoDgP3cJwbCJOfuOi9pK -hQZfYHKklRiilJHfK5jZGQ9s3Q/qVpwTTivgaTyI3TAbx8Q9LdXByhVid8PGZbE= +J+S34QIDAQABoD0wOwYJKoZIhvcNAQkOMS4wLDAqBgNVHREEIzAhggdteXNxbDgw +ggZteXNxbDKCBm15c3FsM4IGbXlzcWw0MA0GCSqGSIb3DQEBCwUAA4IBAQCEZ/qm +4YVvyLfzRvfJ3uo0c0SUzaBnfP21u+X1//93IC1So/AhbFDf/0BlMM1Q3HELl/+Z +HfWwoLSUu3dtbnZIlTp/74GYpWnsVtdHImzqbv361mH+D+Lv09iKX+WNZRtGi2wR +WUgU11m1tgQBlcen2L0NJu7iZ5893nCWzAx8WygQjnD8l95l420KCNQWcQDk6khR +b6ohX2FC7IGf4yNg+DCNVUcmQb3X2zcSMys6aeH2cywJx+bopeXC0O5kcOZSi5Pr +oluzgRsqdD+S6jETSW+vEIqu6H/UJp123SQDiX/YzQG4jtNkRcVSp8UrQWnz7aC1 +tBur9v5Y/j0GHS8g -----END CERTIFICATE REQUEST----- From c7d5dc2620b5a5055c3293323b003ea1de4356c6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 12:57:32 +0000 Subject: [PATCH 04/81] Backport #124453 to 26.8: Add S3 bucket to `remote_url_allow_hosts` --- .../_server_settings_outside_source.mdx | 10 + programs/server/config.xml | 10 + src/Backups/BackupIO_S3.cpp | 2 +- src/Common/RemoteHostFilter.cpp | 49 ++-- src/Common/RemoteHostFilter.h | 7 +- src/Common/tests/gtest_remote_host_filter.cpp | 151 +++++++++++ src/Databases/DatabaseS3.cpp | 2 +- src/Disks/DiskFromAST.cpp | 26 +- .../ObjectStorages/ObjectStorageFactory.cpp | 25 +- .../ObjectStorages/ObjectStorageFactory.h | 6 +- .../RegisterDiskObjectStorage.cpp | 8 +- src/IO/S3/Client.cpp | 6 +- src/IO/S3/URI.cpp | 10 + src/IO/S3/URI.h | 7 + src/IO/S3/tests/gtest_s3_uri.cpp | 70 +++++ src/QueryPipeline/DistributedPlanExecutor.cpp | 2 +- .../ObjectStorage/S3/Configuration.cpp | 2 +- .../remote_url_allow_hosts_s3_bucket.xml | 5 + ...mote_url_allow_hosts_s3_bucket_no_port.xml | 5 + .../test_remote_host_filter_s3_bucket.py | 247 ++++++++++++++++++ ..._custom_disk_error_hides_secrets.reference | 1 + .../05340_custom_disk_error_hides_secrets.sql | 7 + 22 files changed, 610 insertions(+), 48 deletions(-) create mode 100644 src/Common/tests/gtest_remote_host_filter.cpp create mode 100644 tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket.xml create mode 100644 tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts_s3_bucket_no_port.xml create mode 100644 tests/integration/test_backup_restore_s3/test_remote_host_filter_s3_bucket.py create mode 100644 tests/queries/0_stateless/05340_custom_disk_error_hides_secrets.reference create mode 100644 tests/queries/0_stateless/05340_custom_disk_error_hides_secrets.sql diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 23c3fedd9bc7d..37f57886ab2e8 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1690,11 +1690,21 @@ When adding a host with the `\` xml tag: - if the host is specified as an IP address, then it is checked as specified in the URL. For example: `[2a02:6b8:a::a]`. - if there are redirects and support for redirects is enabled, then every redirect (the location field) is checked. +When adding a regular expression with the `\` xml tag, the RE2 engine is used and the whole host is matched. For example: `^.*\.clickhouse\.com$` + +When adding a single S3 bucket with the `\` xml tag, as `host/bucket` or `host:port/bucket`: +- it is useful when the endpoint is shared by many parties, for example the path-style AWS endpoint `s3.us-west-2.amazonaws.com`, and allowing the whole host would allow every bucket on it. +- the bucket is taken from the parsed S3 URL, so the entry matches both `https://s3.us-west-2.amazonaws.com/my-bucket/...` and `https://my-bucket.s3.us-west-2.amazonaws.com/...`. +- it applies to S3 URLs only: the `s3` table functions and engines, the `S3` database engine, `BACKUP`/`RESTORE` to `S3`, and S3 disks created in SQL with the `disk` function. +- a permanent (301) redirect returned by S3 is checked against the bucket entries as well, but a temporary (307) redirect is checked by host only, so it is rejected when the list contains only `\` entries. + For example: ```sql clickhouse.com + ^.*\.clickhouse\.com$ + s3.us-west-2.amazonaws.com/my-bucket ``` diff --git a/programs/server/config.xml b/programs/server/config.xml index 40eb34dd922c6..e3b08e4941a28 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1072,6 +1072,16 @@ Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter (forgetting to do so is a common source of error). --> + + + 3600 + diff --git a/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml new file mode 100644 index 0000000000000..b62bb5c5623e8 --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml @@ -0,0 +1,5 @@ + + + allowed.example.com + + diff --git a/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py new file mode 100644 index 0000000000000..89d8ade4bc71b --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py @@ -0,0 +1,138 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) + +# 26.8.26.12 does not check SQL disks against `remote_url_allow_hosts`. The objects are created there with a list +# that does not name MinIO, then the server restarts on the current binary. +# Two instances: a failed check on the database metadata disk would stop the server. +node = cluster.add_instance( + "node", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.8.26.12", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) +node_db = cluster.add_instance( + "node_db", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.8.26.12", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) + +WARNING_IN_LOG = "is not allowed by `remote_url_allow_hosts`" + + +def s3_disk(path, disk_type="s3"): + return ( + f"disk(type = '{disk_type}', endpoint = 'http://minio1:9001/root/data/{path}/', " + f"access_key_id = 'minio', secret_access_key = '{minio_secret_key}')" + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_table_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node.query( + f"CREATE TABLE t (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('upgrade_disk')}" + ) + node.query("INSERT INTO t VALUES (1), (2), (3)") + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + # Detached permanently before the upgrade, so its disk is first built by the short ATTACH on the new binary. + node.query( + f"CREATE TABLE t_detached (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('detached_disk')}" + ) + node.query("INSERT INTO t_detached VALUES (4), (5)") + node.query("DETACH TABLE t_detached PERMANENTLY") + # Dropped before the upgrade and still in the drop queue: the new binary rebuilds it at start from + # metadata_dropped under FORCE_RESTORE, which is the path UNDROP relies on. + node.query( + f"CREATE TABLE t_dropped (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('dropped_disk')}" + ) + node.query("INSERT INTO t_dropped VALUES (6), (7)") + node.query("DROP TABLE t_dropped") + + node.restart_with_latest_version() + + # Loaded from existing metadata: reported, not rejected. + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + assert node.contains_in_log(WARNING_IN_LOG) + assert ( + node.query( + "SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%minio1:9001/root%'" + ) + == "1\n" + ) + # Two disks were built at start and reported: the live table's and the dropped table's. + assert ( + node.query( + "SELECT countSubstrings(message, '->') FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%'" + ) + == "2\n" + ) + node.query("UNDROP TABLE t_dropped") + assert node.query("SELECT id FROM t_dropped ORDER BY id") == "6\n7\n" + + # Only loading existing metadata is exempt. A user ATTACH is checked like a CREATE, in the short form too: + # the first build of the parked table's disk is rejected, like every other disk check on a short ATTACH. + error = node.query_and_get_error("ATTACH TABLE t_detached") + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"CREATE TABLE t_new (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('new_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"ATTACH TABLE t_attached UUID '{uuid.uuid4()}' (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('attached_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + + # The operator adds the entry that the warning names; the parked table attaches without a restart. + node.replace_config( + "/etc/clickhouse-server/config.d/remote_url_allow_hosts_without_minio.xml", + "allowed.example.comminio1:9001/root", + ) + node.query("SYSTEM RELOAD CONFIG") + node.query("ATTACH TABLE t_detached") + assert node.query("SELECT id FROM t_detached ORDER BY id") == "4\n5\n" + + +def test_database_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node_db.query( + f"CREATE DATABASE db ENGINE = Atomic SETTINGS disk = {s3_disk('upgrade_db_disk', 's3_plain_rewritable')}" + ) + node_db.query("CREATE TABLE db.t (id UInt64) ENGINE = MergeTree ORDER BY id") + node_db.query("INSERT INTO db.t VALUES (1), (2), (3)") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + + node_db.restart_with_latest_version() + + # The server starts and the database loads; the disk is reported. + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + assert node_db.contains_in_log(WARNING_IN_LOG) + + # DETACH and ATTACH reuse the metadata disk built at server start, so no check runs. + node_db.query("DETACH DATABASE db") + node_db.query("ATTACH DATABASE db") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" From 49368dd16cc2cce474e23a905d16c2cc3b43f612 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 16:49:56 +0000 Subject: [PATCH 08/81] Backport #125487 to 26.8: Fix CapnProto schema depth check bypass through comments and backtick text lines --- src/Formats/CapnProtoSchema.cpp | 45 +++++++++++---- ...apnproto_deep_schema_lexer_forms.reference | 15 +++++ ...05366_capnproto_deep_schema_lexer_forms.sh | 57 +++++++++++++++++++ 3 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.reference create mode 100755 tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.sh diff --git a/src/Formats/CapnProtoSchema.cpp b/src/Formats/CapnProtoSchema.cpp index 977f07070a5a6..9bbbf7047d3ba 100644 --- a/src/Formats/CapnProtoSchema.cpp +++ b/src/Formats/CapnProtoSchema.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -80,8 +81,10 @@ void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const Str size_t depth = 0; /// The last identifier seen before the current position, to recognize `import "..."`. String last_word; - /// Whether `last_word` has already been terminated by whitespace. + /// Whether `last_word` has already been terminated by whitespace, a comment or a byte order mark, + /// any of which Cap'n Proto allows between `import` and its path. bool last_word_finished = false; + static constexpr std::string_view utf8_bom = "\xEF\xBB\xBF"; for (size_t i = 0; i < content.size(); ++i) { @@ -90,20 +93,41 @@ void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const Str /// Cap'n Proto has line comments and text literals; brackets inside them are not nesting. if (c == '#') { - last_word.clear(); - last_word_finished = false; + last_word_finished = !last_word.empty(); i = content.find('\n', i); if (i == String::npos) break; } - else if (c == '"') + else if (c == '"' || c == '`') { String literal; - for (++i; i < content.size() && content[i] != '"'; ++i) + if (c == '`') + { + /// A backtick starts a text literal that runs to the end of the line, the line break included. + for (++i; i < content.size() && content[i] != '\r' && content[i] != '\n'; ++i) + literal += content[i]; + literal += '\n'; + } + else { - if (content[i] == '\\' && i + 1 < content.size()) - ++i; - literal += content[i]; + kj::parse::IteratorInput input(content.data() + i, content.data() + content.size()); + auto parsed = kj::parse::doubleQuotedString(input); + if (const auto * text = kj::_::readMaybe(parsed)) + { + /// Lexed as Cap'n Proto does, so the escape sequences are decoded in the same way. + literal.assign(text->begin(), text->size()); + i = input.getPosition() - content.data() - 1; + } + else + { + /// Not a text literal, e.g. a `0x"..."` data literal with a line break. + for (++i; i < content.size() && content[i] != '"'; ++i) + { + if (content[i] == '\\' && i + 1 < content.size()) + ++i; + literal += content[i]; + } + } } if (last_word == "import" && !literal.empty()) @@ -126,10 +150,11 @@ void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const Str } last_word += c; } - else if (isWhitespaceASCII(c)) + else if (isWhitespaceASCII(c) || content.compare(i, utf8_bom.size(), utf8_bom) == 0) { - /// Whitespace separates `import` from the path, so it must not forget the keyword. last_word_finished = !last_word.empty(); + if (!isWhitespaceASCII(c)) + i += utf8_bom.size() - 1; } else { diff --git a/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.reference b/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.reference new file mode 100644 index 0000000000000..11475ac2e46c9 --- /dev/null +++ b/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.reference @@ -0,0 +1,15 @@ +import_after_comment +1 +import_after_bom +1 +import_escaped +1 +import_backtick +1 +quote_in_backtick +1 +hash_in_backtick +1 +data_literal_with_line_break +1 +data Int32 diff --git a/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.sh b/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.sh new file mode 100755 index 0000000000000..489cba91a4624 --- /dev/null +++ b/tests/queries/0_stateless/05366_capnproto_deep_schema_lexer_forms.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The CapnProto schema nesting depth check must see the same tokens as the Cap'n Proto lexer: comments and byte order +# marks between `import` and its path, escape sequences in a quoted path, and backtick text lines. + +guard_message_count() { + local output count + output=$(cat) + count=$(grep -c -F 'nested too deeply' <<< "$output") || true + echo "$count" + [ "$count" = 1 ] || echo "FAIL: the depth guard did not fire, got: $(head -c 1000 <<< "$output" | tr '\n' ' ')" +} + +SCHEMA_DIR="$(mktemp -d "${CLICKHOUSE_TMP}/capnproto_schemas_XXXXXX")" +SCHEMA_DIR="$(cd "${SCHEMA_DIR}" && pwd)" +trap 'rm -rf "${SCHEMA_DIR}"' EXIT + +python3 - "${SCHEMA_DIR}" <<'PY' +import sys +directory = sys.argv[1] +nested = 'List(' * 20000 + 'Int32' + ')' * 20000 + +def schema(name, text): + with open(directory + '/' + name, 'w', encoding='utf-8', newline='') as f: + f.write(text) + +schema('deep.capnp', '@0x844f048b15c12dac;\nstruct D { data @0 :' + nested + '; }\n') +# The value of a backtick text literal ends with a line break, also when it is the path of an import. +schema('deep_backtick.capnp\n', '@0x844f048b15c12dad;\nstruct D { data @0 :' + nested + '; }\n') + +schema('import_after_comment.capnp', '@0x844f048b15c12dab;\nusing D = import # c\n"deep.capnp";\nstruct M { data @0 :D.D; }\n') +schema('import_after_bom.capnp', '@0x844f048b15c12dab;\nusing D = import\ufeff"deep.capnp";\nstruct M { data @0 :D.D; }\n') +schema('import_escaped.capnp', '@0x844f048b15c12dab;\nusing D = import "deep\\x2ecapnp";\nstruct M { data @0 :D.D; }\n') +schema('import_backtick.capnp', '@0x844f048b15c12dab;\nusing D = import `deep_backtick.capnp\n;\nstruct M { data @0 :D.D; }\n') +schema('quote_in_backtick.capnp', '@0x844f048b15c12dab;\nconst a :Text = `"\n;\nstruct M { data @0 :' + nested + '; }\n') +schema('hash_in_backtick.capnp', '@0x844f048b15c12dab;\nconst a :Text = `#\r;struct M { data @0 :' + nested + '; }\n') +schema('brackets_in_backtick.capnp', '@0x844f048b15c12dab;\nconst a :Text = `' + '(' * 1001 + '\n;\nstruct M { data @0 :Int32; }\n') +schema('data_literal_with_line_break.capnp', '@0x844f048b15c12dab;\nconst a :Data = 0x"62\n61";\nstruct M { data @0 :' + nested + '; }\n') +PY + +for name in import_after_comment import_after_bom import_escaped import_backtick quote_in_backtick hash_in_backtick data_literal_with_line_break +do + echo "${name}" + ${CLICKHOUSE_LOCAL} --logger.console=0 --query " + DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/${name}.capnp:M' + " 2>&1 | guard_message_count +done + +# Brackets in a backtick text line are text, not nesting. +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/brackets_in_backtick.capnp:M' +" From b6b838ba572701496f3c3e5dffe4615bf25307e9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 16:52:12 +0000 Subject: [PATCH 09/81] Update autogenerated version to 26.8.27.9 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 372b3aaad469a..797c1fa466218 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54539) +SET(VERSION_REVISION 54540) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 27) -SET(VERSION_GITHASH a9f78f125b269c5cb9013dc793330e45a0bc316b) -SET(VERSION_DESCRIBE v26.8.27.1-lts) -SET(VERSION_STRING 26.8.27.1) +SET(VERSION_PATCH 28) +SET(VERSION_GITHASH 551899e257e06d9e3c977481944b69695adb3c32) +SET(VERSION_DESCRIBE v26.8.28.1-lts) +SET(VERSION_STRING 26.8.28.1) # end of autochange From 4a1979a156df842f347606e03398204a84d23c2e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 20:46:19 +0000 Subject: [PATCH 10/81] Backport #125661 to 26.8: Validate `rabbitmq_routing_key_list` in the `RabbitMQ` engine --- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 17 +++++++++++++++++ ...5322_rabbitmq_invalid_routing_keys.reference | 0 .../05322_rabbitmq_invalid_routing_keys.sql | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.reference create mode 100644 tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.sql diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 7aead829181d5..947b3bf26d615 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -149,6 +149,23 @@ StorageRabbitMQ::StorageRabbitMQ( std::find_if(queue_settings_list.begin(), queue_settings_list.end(), [](const String & name) { return name.starts_with(deadletter_exchange_setting); }); + /// The first routing key is used unconditionally for binding and publishing. + /// An empty list is meaningful (e.g. `fanout` ignores the routing key, `rabbitmq_queue_consume` + /// does not bind at all), so treat it as a single empty routing key. + if (routing_keys.empty()) + routing_keys.emplace_back(); + + /// For the `headers` exchange every routing key is a `key=value` pair. + if (exchange_type == AMQP::ExchangeType::headers) + { + for (const auto & header : routing_keys) + { + if (!header.contains('=')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Invalid routing key '{}' for the `headers` exchange type: expected `key=value`", header); + } + } + const auto & config = getContext()->getConfigRef(); std::pair parsed_address; diff --git a/tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.reference b/tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.sql b/tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.sql new file mode 100644 index 0000000000000..82492dd9b117d --- /dev/null +++ b/tests/queries/0_stateless/05322_rabbitmq_invalid_routing_keys.sql @@ -0,0 +1,16 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: the RabbitMQ engine is not built in the fast test. + +-- Malformed routing keys are rejected before connecting to the broker, +-- so the host below does not need to exist. + +-- A routing key without `=` for the `headers` exchange used to cause an out-of-bounds read. +CREATE TABLE t_rmq_headers (x String) ENGINE = RabbitMQ +SETTINGS rabbitmq_host_port = 'localhost:5672', rabbitmq_username = 'u', rabbitmq_password = 'p', + rabbitmq_exchange_name = 'e', rabbitmq_exchange_type = 'headers', + rabbitmq_routing_key_list = 'format,logs', rabbitmq_format = 'JSONEachRow'; -- { serverError BAD_ARGUMENTS } + +CREATE TABLE t_rmq_headers (x String) ENGINE = RabbitMQ +SETTINGS rabbitmq_host_port = 'localhost:5672', rabbitmq_username = 'u', rabbitmq_password = 'p', + rabbitmq_exchange_name = 'e', rabbitmq_exchange_type = 'headers', + rabbitmq_routing_key_list = 'x-match=all,format', rabbitmq_format = 'JSONEachRow'; -- { serverError BAD_ARGUMENTS } From e0e7f876c4282e10201627065df98f3b0e8e1975 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 20:49:08 +0000 Subject: [PATCH 11/81] Backport #125512 to 26.8: Never stop the server process from the line editor of the embedded client --- contrib/replxx | 2 +- src/Client/ReplxxLineReader.cpp | 8 ++++-- tests/integration/test_webterminal/test.py | 30 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/contrib/replxx b/contrib/replxx index 4ab25d428dd0b..00cd994b363c3 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 4ab25d428dd0bfc0590cac0d15e1fb872794dd62 +Subproject commit 00cd994b363c3fc32f80d9d266e4d2acb4159abf diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 6a564e5e19650..9ef83c3974eac 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -385,7 +385,10 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) } } - rx.install_window_change_handler(); + /// The handler of `SIGWINCH` is process-wide. In the embedded mode the process is the server, + /// and the pseudoterminal is not its controlling terminal, so the signal is never delivered. + if (!options.embedded_mode) + rx.install_window_change_handler(); auto callback = [this] (const String & context, size_t context_size) { @@ -540,7 +543,8 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) rx.bind_key(Replxx::KEY::meta('r'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_SEEDED_INCREMENTAL_SEARCH, code); }); /// We don't want the default, "suspend" behavior, it confuses people. - if (options.ignore_shell_suspend) + /// In the embedded mode, it would stop the whole server process with `SIGSTOP`. + if (options.ignore_shell_suspend || options.embedded_mode) rx.bind_key_internal(replxx::Replxx::KEY::control('Z'), "insert_character"); auto commit_action = [this](char32_t code) diff --git a/tests/integration/test_webterminal/test.py b/tests/integration/test_webterminal/test.py index 8141d45ba4e14..6f5e76944392a 100644 --- a/tests/integration/test_webterminal/test.py +++ b/tests/integration/test_webterminal/test.py @@ -570,6 +570,36 @@ def test_tab_completion_respects_session_user(): instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") +def test_ctrl_z_does_not_stop_server(): + """Ctrl-Z in the embedded client must not suspend the server process. + + The embedded client runs inside the server, so the "suspend" action of the + line editor (`raise(SIGSTOP)`) would stop the whole server. It is disabled by + the key binding, but the incremental history search (Ctrl-R) used to handle + Ctrl-Z by itself, bypassing the binding. + """ + sock = _open_ws(instance.ip_address, 8123) + try: + _ws_send_text(sock, json.dumps({"type": "auth", "user": "default", "password": ""})) + output = _read_pty_until(sock, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # Ctrl-Z in the main loop, then Ctrl-R (history search) followed by Ctrl-Z. + _ws_send_binary(sock, b"\x1a") + _ws_send_binary(sock, b"\x12\x1a") + _read_pty_until(sock, timeout=1) + + response = instance.http_request("ping", method="GET", timeout=10) + assert response.status_code == 200 + + # The session is still usable: Ctrl-U clears the line, then run a query. + _ws_send_binary(sock, b"\x15SELECT 1234 + 1\r") + output = _read_pty_until(sock, timeout=10, marker="1235") + assert "1235" in output, f"the embedded client does not respond: {output!r}" + finally: + sock.close() + + def test_session_keeps_user_settings(): """The embedded client must keep the settings of the authenticated user. From b9da51e8ac9584911289a2df73df0095de6fad81 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 21:06:17 +0000 Subject: [PATCH 12/81] Backport #125575 to 26.8: Do not eliminate aggregators of `GROUPING SETS` keys with `group_by_use_nulls` --- .../Passes/AggregateFunctionOfGroupByKeysPass.cpp | 7 +++++++ ...roup_by_keys_grouping_sets_use_nulls.reference | 3 +++ ...s_of_group_by_keys_grouping_sets_use_nulls.sql | 15 +++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.reference create mode 100644 tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.sql diff --git a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp index aed48de27380b..3f3d1e6cafbbe 100644 --- a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp @@ -13,6 +13,7 @@ namespace DB { namespace Setting { + extern const SettingsBool group_by_use_nulls; extern const SettingsBool optimize_aggregators_of_group_by_keys; } @@ -52,6 +53,12 @@ class EliminateFunctionVisitor : public InDepthQueryTreeVisitorWithContextisGroupByWithGroupingSets() && getSettings()[Setting::group_by_use_nulls]) + { + /// With `group_by_use_nulls` the keys of grouping sets become `Nullable` in the aggregation's output, + /// while the argument of the aggregate function keeps its type, so the replacement would not match it. + group_by_keys_stack.push_back({}); + } else { QueryTreeNodePtrWithHashSet group_by_keys; diff --git a/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.reference b/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.reference new file mode 100644 index 0000000000000..67a4a778f01c6 --- /dev/null +++ b/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.reference @@ -0,0 +1,3 @@ +4950 100 +4950 100 +5049 101 diff --git a/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.sql b/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.sql new file mode 100644 index 0000000000000..8051fe697d5bd --- /dev/null +++ b/tests/queries/0_stateless/05386_aggregators_of_group_by_keys_grouping_sets_use_nulls.sql @@ -0,0 +1,15 @@ +-- With `group_by_use_nulls` the keys of grouping sets are `Nullable` in the output of the aggregation, +-- so `optimize_aggregators_of_group_by_keys` must not replace `any` of a key with the key itself: +-- the outer `sum` read the `Nullable` column as a plain one, which gave a wrong result or an exception. + +DROP TABLE IF EXISTS t_aggregators_of_keys_use_nulls; +CREATE TABLE t_aggregators_of_keys_use_nulls (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a; +INSERT INTO t_aggregators_of_keys_use_nulls SELECT number, number % 100 FROM numbers(10000); + +SET group_by_use_nulls = 1, optimize_aggregators_of_group_by_keys = 1; + +SELECT sum(s), count() FROM (SELECT b AS k, anyHeavy(b) AS s FROM t_aggregators_of_keys_use_nulls GROUP BY GROUPING SETS ((k))); +SELECT sum(s), count() FROM (SELECT b AS k, any(b) AS s FROM t_aggregators_of_keys_use_nulls GROUP BY GROUPING SETS ((k))); +SELECT sum(s), count() FROM (SELECT b AS k, max(b) AS s FROM t_aggregators_of_keys_use_nulls GROUP BY GROUPING SETS ((k), ())); + +DROP TABLE t_aggregators_of_keys_use_nulls; From e5d73bbd8d2ef655cc2821b67c9216835ceeb9ab Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 22:24:47 +0000 Subject: [PATCH 13/81] Backport #125712 to 26.8: Stream-parse the `.backup` metadata file --- src/Backups/BackupImpl.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 7f3912d748725..74deb09197fd9 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #if CLICKHOUSE_CLOUD && USE_SSL #include #endif +#include #include #include @@ -672,9 +674,6 @@ void BackupImpl::readBackupMetadata() in = reader->readFile(".backup"); } - String str; - readStringUntilEOF(str, *in); - num_files = 0; total_size = 0; num_entries = 0; @@ -865,9 +864,18 @@ void BackupImpl::readBackupMetadata() /// ordinary element; with it the handler sees "x:contents" and ignores it (writeBackupMetadata never /// emits namespaces, so this only rejects hand-crafted manifests). xml_parser.setFeature(Poco::XML::XMLReader::FEATURE_NAMESPACE_PREFIXES, true); + /// The metadata is parsed while it is read, in chunks, instead of being read into one string first: it has + /// a record per file and can take gigabytes for a backup of many files. + ReadBuffer & metadata_in = *in; + /// The parser reads the stream from start to end and never seeks, so the stream doesn't need the size. + StdIStreamFromReadBuffer metadata_stream(std::move(in), /*size=*/ 0); + /// `std::istream` swallows an exception thrown by the buffer and only sets `badbit`, which the parser takes + /// for the end of the data. Rethrow it instead, so a read error isn't reported as a malformed manifest. + metadata_stream.exceptions(std::ios::badbit); + Poco::XML::InputSource metadata_source(metadata_stream); try { - xml_parser.parseMemoryNP(str.data(), str.size()); + xml_parser.parse(&metadata_source); } catch (...) { @@ -887,7 +895,7 @@ void BackupImpl::readBackupMetadata() if (!contents_seen) throw Exception(ErrorCodes::BACKUP_DAMAGED, "Backup {}: Metadata has no ", backup_name_for_logging); - uncompressed_size = size_of_entries + str.size(); + uncompressed_size = size_of_entries + metadata_in.count(); #if CLICKHOUSE_CLOUD && USE_SSL /// A backup written with an encryption config file next to it carries the TDE key information of that From 098343e9a9f3aa4f873bcf29a24930379b8af438 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 23:57:40 +0000 Subject: [PATCH 14/81] Backport #125774 to 26.8: Bump `vectorscan` from 5.4.11 to 5.4.13 --- .gitmodules | 2 +- contrib/vectorscan | 2 +- contrib/vectorscan-cmake/CMakeLists.txt | 15 +- contrib/vectorscan-cmake/common/hs_version.h | 4 +- .../rageled_files/aarch64/Parser.cpp | 4073 ++++++------ .../rageled_files/aarch64/control_verbs.cpp | 50 +- .../rageled_files/amd64/Parser.cpp | 5537 +++++++++-------- .../rageled_files/amd64/control_verbs.cpp | 92 +- 8 files changed, 4917 insertions(+), 4858 deletions(-) diff --git a/.gitmodules b/.gitmodules index cee2d45690c0d..26fec9523a636 100644 --- a/.gitmodules +++ b/.gitmodules @@ -208,7 +208,7 @@ url = https://github.com/ClickHouse/nats.c [submodule "contrib/vectorscan"] path = contrib/vectorscan - url = https://github.com/ClickHouse/vectorscan.git + url = https://github.com/ClickHouse/vectorscan [submodule "contrib/llvm-project"] path = contrib/llvm-project url = https://github.com/ClickHouse/llvm-project diff --git a/contrib/vectorscan b/contrib/vectorscan index e6993b7003a19..4f863adbc2aa1 160000 --- a/contrib/vectorscan +++ b/contrib/vectorscan @@ -1 +1 @@ -Subproject commit e6993b7003a19806902fa69ac57dcc2df826b946 +Subproject commit 4f863adbc2aa1129d473483d1a70134da07b5074 diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 2d8a05af15024..ab06d8a8ac601 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -37,8 +37,8 @@ set (SRCS "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/fdr/teddy_avx2.c" + "${LIBRARY_DIR}/src/fdr/teddy.cpp" + "${LIBRARY_DIR}/src/fdr/teddy_fat.cpp" "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" @@ -238,11 +238,20 @@ set (SRCS # weird reasons different constants in the output. # # Also, please use upstream versions of colm and ragel, the packages in Ubuntu 22.04 seem to produce wrong output on ARM. +# +# The generated scanners read the input through `const char *` but compare it against `unsigned char` tables (the .rl files declare +# `alphtype unsigned char`), so they must be compiled with `-funsigned-char`, as the original build system does, otherwise bytes >= 0x80 +# never match on platforms where `char` is signed. if (ARCH_AMD64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" ) + set_source_files_properties( + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" + COMPILE_FLAGS -funsigned-char + ) elseif (ARCH_AARCH64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" @@ -251,7 +260,7 @@ elseif (ARCH_AARCH64) set_source_files_properties( "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp" - COMPILE_FLAGS -Wno-c++11-narrowing + COMPILE_FLAGS "-funsigned-char -Wno-c++11-narrowing" ) endif() diff --git a/contrib/vectorscan-cmake/common/hs_version.h b/contrib/vectorscan-cmake/common/hs_version.h index 3d266484095e4..6b69038d3f09c 100644 --- a/contrib/vectorscan-cmake/common/hs_version.h +++ b/contrib/vectorscan-cmake/common/hs_version.h @@ -32,12 +32,12 @@ /** * A version string to identify this release of Hyperscan. */ -#define HS_VERSION_STRING "5.4.11 2024-07-04" +#define HS_VERSION_STRING "5.4.13 2026-08-23" #define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0) #define HS_MAJOR 5 #define HS_MINOR 4 -#define HS_PATCH 11 +#define HS_PATCH 13 #endif /* HS_VERSION_H_C6428FAF8E3713 */ diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp index fd58ead412837..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,17 +384,18 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { @@ -500,14 +501,14 @@ static const short _regex_actions[] = { 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1907, 1946, 1948, - 1950, 1952, 1954, 1955, 1955, 1956, 1957, 1964, - 1970, 1976, 1978, 1980, 1982, 1985, 1987, 1998, - 1999, 2001, 2003, 2005, 2016, 2017, 2019, 2021, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { + static const unsigned char _regex_trans_keys[] = { 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, @@ -744,24 +745,24 @@ static const short _regex_actions[] = { 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, - 83u, 90u, 45u, 91u, 92u, 93u, 128u, 191u, - 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 46u, 58u, 61u, 48u, 68u, 69u, 72u, 76u, - 78u, 80u, 81u, 83u, 85u, 86u, 87u, 97u, - 98u, 99u, 100u, 101u, 102u, 103u, 104u, 108u, - 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, - 119u, 120u, 49u, 55u, 56u, 57u, 65u, 90u, - 105u, 122u, 48u, 55u, 48u, 55u, 48u, 55u, - 48u, 55u, 123u, 123u, 123u, 123u, 48u, 57u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, - 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, - 128u, 191u, 128u, 191u, 128u, 191u, 92u, 93u, - 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, - 224u, 239u, 240u, 247u, 248u, 255u, 69u, 128u, - 191u, 128u, 191u, 128u, 191u, 92u, 128u, 191u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 69u, 128u, 191u, 128u, 191u, 128u, 191u, 41u, - 10u, 0u + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { @@ -867,11 +868,11 @@ static const short _regex_actions[] = { 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 3, 31, 0, 0, - 0, 0, 1, 0, 1, 1, 1, 0, - 0, 0, 0, 0, 3, 2, 1, 1, - 0, 0, 0, 1, 1, 0, 0, 0, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { @@ -977,11 +978,11 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 0, 4, 1, 1, - 1, 1, 0, 0, 0, 0, 3, 3, - 3, 1, 1, 1, 0, 0, 5, 0, - 1, 1, 1, 5, 0, 1, 1, 1, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { @@ -1087,11 +1088,11 @@ static const short _regex_actions[] = { 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2553, 2589, 2591, - 2593, 2595, 2597, 2599, 2600, 2602, 2604, 2609, - 2613, 2617, 2619, 2621, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { @@ -1412,8 +1413,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1422,12 +1423,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, -1, 0, -1, 0, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, 0, - -1, 0, -1, 0, -1, -1, 0, 0, - 0, 0, 0, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,7 +1535,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { @@ -1856,7 +1857,7 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2576, 2577, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, @@ -1865,12 +1866,12 @@ static const short _regex_actions[] = { 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, - 2642, 2643, 2645, 2646, 2648, 2649, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2674, 2675, - 2677, 2678, 2680, 2681, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2697, 2698, 2700, - 2701, 2703, 2704, 2706, 2707, 2708, 2709, 2710, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,7 +1978,7 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { @@ -2298,8 +2299,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2308,12 +2309,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 2, - 1, 2, 1, 2, 1, 1, 2, 2, - 2, 2, 2, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,7 +2421,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { @@ -2744,8 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2754,15 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 0, 0, 1, 0, 0, 1, 0, - 0, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,7 +2870,7 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { @@ -3144,9 +3145,9 @@ static const short _regex_actions[] = { 732, 733, 819, 731, 732, 733, 731, 735, 819, 819, 735, 819, 819, 736, 736, 736, 819, 819, 819, 819, 819, 739, 819, 819, - 819, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, 746, 747, 746, 759, 760, 746, 761, 762, 763, 746, 782, 746, 746, 746, 746, 746, 784, 746, 785, 746, 786, 746, 746, 746, @@ -3193,26 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 820, 821, 819, 819, 819, 819, 833, - 819, 834, 819, 835, 819, 819, 819, 563, - 566, 731, 819, 822, 819, 819, 819, 819, - 819, 826, 819, 819, 819, 819, 819, 819, - 819, 827, 819, 819, 819, 819, 819, 819, - 819, 828, 829, 819, 819, 819, 819, 819, - 819, 830, 824, 819, 819, 819, 819, 823, - 819, 819, 819, 825, 819, 819, 819, 819, - 819, 819, 734, 819, 819, 819, 832, 831, - 831, 831, 819, 819, 819, 819, 819, 736, - 736, 736, 819, 819, 819, 819, 819, 737, - 819, 819, 738, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 839, 838, 838, - 838, 840, 838, 841, 838, 842, 838, 838, - 838, 838, 838, 838, 838, 838, 838, 740, - 838, 838, 741, 838, 844, 843, 843, 843, - 845, 843, 846, 843, 847, 843, 843, 843, - 843, 843, 843, 843, 843, 843, 743, 843, - 843, 744, 843, 848, 848, 849, 849, 0, - 746, 746, 746, 746, 746, 746, 746, 746, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3220,7 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 827, 631, 731, 723, 45, 903, 903, 897, - 45, 912, 900, 903, 729, 741, 0, 45, - 45, 923, 923, 923, 923, 737, 45, 765, - 747, 0, 0, 0, 0, 841, 743, 3, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, 3, 3, 3, 841, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 761, - 839, 761, 3, 3, 3, 3, 839, 759, - 3, 870, 3, 3, 839, 761, 3, 870, - 3, 3, 839, 3, 3, 3, 3, 839, - 0, 3, 3, 3, 3, 839, 751, 0, - 749, 839, 755, 753, 839, 0, 757, 3, - 3, 3, 3, 839, 0, 0, 0, 0, - 839, 757, 0, 0, 0, 0, 839, 757, - 3, 3, 3, 3, 839, 745, 0, 841, - 725, 727, 27, 27, 27, 27, 841, 743, - 3, 3, 3, 3, 841, 763, 0, 841, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 841, 673, 3, 3, 3, 3, 841, - 0, 0, 0, 0, 841, 27, 27, 27, - 27, 841, 725, 0, 727, 25, 25, 25, - 25, 841, 0, 0, 829, 0, 0, 0, - 603, 0, 0, 0, 0, 0, 0, 829, - 0, 0, 0, 601, 0, 0, 0, 0, - 0, 0, 829, 0, 0, 0, 603, 0, - 0, 0, 0, 675, 0, 17, 831, 675, - 0, 7, 3, 870, 3, 3, 831, 17, - 831, 663, 17, 831, 663, 17, 831, 663, - 831, 3, 661, 870, 3, 3, 831, 3, - 661, 870, 3, 3, 831, 3, 661, 3, - 3, 3, 831, 3, 665, 3, 3, 3, - 831, 3, 3, 3, 3, 837, 671, 3, - 3, 3, 3, 837, 3, 3, 3, 3, - 837, 669, 3, 3, 3, 3, 837, 3, - 3, 3, 3, 837, 3, 667, 3, 3, - 3, 837, 0, 833, 677, 0, 833, 679, - 0, 0, 0, 835, 0, 29, 11, 845, - 31, 13, 845, 0, 13, 845, 843, 769, - 843, 843, 0, 843, 843, 771, 843, 53, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 0, 0, 539, 0, 0, 0, 377, - 0, 0, 0, 0, 0, 0, 539, 0, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 539, 0, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 435, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 381, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 385, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 389, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 393, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 397, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 401, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 405, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 409, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 413, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 417, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 421, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 425, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 429, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 433, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 379, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 383, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 387, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 391, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 395, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 399, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 403, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 407, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 411, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 415, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 419, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 423, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 427, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 431, 0, 0, 0, 539, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, 0, 0, 377, 0, 0, 0, 0, 0, - 533, 471, 0, 533, 475, 0, 0, 0, - 535, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 775, 777, 773, 777, - 0, 777, 45, 777, 45, 777, 773, 777, - 0, 0, 779, 0, 0, 0, 0, 821, - 3, 3, 3, 3, 821, 1, 3, 1, - 0, 3, 3, 855, 855, 3, 3, 819, - 0, 821, 27, 27, 27, 27, 0, 821, - 745, 0, 821, 735, 733, 3, 3, 3, - 3, 821, 739, 821, 763, 0, 821, 1, - 1, 0, 821, 725, 0, 727, 25, 25, - 25, 25, 821, 615, 613, 783, 619, 617, - 785, 623, 621, 787, 0, 0, 0, 781, - 5, 639, 647, 611, 695, 607, 717, 699, - 715, 683, 683, 0, 605, 713, 691, 683, - 703, 687, 719, 641, 657, 645, 0, 693, - 659, 655, 906, 697, 45, 683, 651, 45, - 0, 653, 689, 649, 683, 701, 685, 7, - 643, 852, 915, 721, 15, 791, 867, 791, - 858, 919, 795, 927, 919, 793, 919, 847, - 709, 711, 681, 0, 7, 0, 849, 17, - 801, 17, 799, 876, 799, 17, 797, 873, - 797, 1, 1, 1, 815, 0, 803, 705, - 707, 45, 19, 23, 21, 805, 882, 894, - 888, 805, 0, 0, 0, 807, 11, 825, - 627, 625, 789, 823, 767, 823, 823, 0, - 823, 823, 0, 823, 55, 45, 45, 53, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,48 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 0, - 0, 0, 531, 5, 489, 439, 457, 493, - 493, 0, 437, 485, 493, 461, 481, 451, - 441, 0, 487, 453, 449, 495, 455, 493, - 445, 45, 0, 447, 483, 443, 493, 459, - 479, 7, 9, 473, 497, 497, 499, 15, - 517, 861, 517, 15, 519, 864, 519, 467, - 469, 477, 0, 521, 463, 465, 45, 19, - 23, 21, 523, 879, 891, 885, 523, 0, - 0, 0, 525, 529, 501, 529, 529, 0, - 529, 529, 0, 529, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 553, 567, 565, 555, 565, 565, 0, - 565, 565, 0, 565, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 571, 585, 583, 573, 583, 583, 0, 583, - 583, 0, 583, 589, 591, 593, 595, 0, - 827, 827, 841, 841, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 841, 841, 841, 841, 841, 841, - 841, 841, 841, 841, 829, 829, 829, 829, - 829, 829, 829, 829, 829, 831, 831, 831, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 837, 837, 837, 837, 837, 837, 833, - 833, 835, 845, 845, 845, 843, 843, 843, - 59, 59, 59, 59, 59, 59, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3690,8 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, + 355, 355, 355, 355, 355, 355, 355, 355, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 533, 533, 535, - 537, 537, 537, 569, 569, 569, 587, 587, - 587, 0, 779, 821, 821, 819, 821, 821, - 821, 821, 821, 821, 821, 821, 783, 785, - 787, 781, 817, 791, 791, 795, 793, 847, - 811, 809, 801, 799, 799, 797, 797, 815, - 803, 813, 805, 805, 807, 825, 789, 823, - 823, 823, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 531, 531, 517, 517, 519, - 519, 513, 527, 521, 515, 523, 523, 525, - 529, 529, 529, 0, 551, 0, 567, 565, - 565, 565, 0, 585, 583, 583, 583, 0, - 0, 0 + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp index 6204fc0a31462..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp @@ -74,7 +74,7 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { + static const unsigned char _ControlVerbs_trans_keys[] = { 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp index ffda15155821d..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,29 +384,30 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { - 0, 0, 2, 4, 6, 7, 29, 37, - 45, 52, 60, 61, 69, 77, 85, 92, - 100, 103, 105, 114, 121, 129, 137, 140, - 146, 154, 157, 164, 171, 179, 186, 190, - 197, 200, 203, 205, 208, 211, 213, 216, - 219, 221, 222, 224, 225, 233, 235, 238, - 241, 242, 250, 258, 266, 274, 281, 289, - 296, 304, 311, 319, 321, 324, 331, 335, - 338, 341, 342, 344, 345, 347, 349, 350, + 0, 0, 1, 23, 31, 39, 46, 54, + 55, 63, 71, 79, 86, 94, 97, 99, + 108, 115, 123, 131, 134, 140, 148, 151, + 158, 165, 173, 180, 184, 191, 194, 197, + 199, 202, 205, 207, 210, 213, 215, 216, + 218, 219, 227, 229, 232, 235, 236, 244, + 252, 260, 268, 275, 283, 290, 298, 305, + 313, 315, 318, 325, 329, 332, 335, 337, + 339, 341, 342, 344, 345, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 369, 370, 371, 372, 373, 374, 375, 376, 377, @@ -468,312 +469,312 @@ static const short _regex_actions[] = { 849, 850, 851, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, - 874, 875, 876, 877, 879, 881, 883, 886, - 889, 891, 906, 909, 912, 914, 928, 933, - 938, 942, 946, 949, 952, 956, 960, 963, - 966, 970, 974, 978, 981, 984, 988, 992, - 996, 1000, 1003, 1006, 1010, 1014, 1018, 1022, - 1025, 1028, 1032, 1036, 1040, 1044, 1047, 1050, - 1054, 1058, 1062, 1066, 1069, 1072, 1076, 1080, - 1084, 1088, 1091, 1094, 1099, 1103, 1107, 1111, - 1114, 1117, 1121, 1125, 1129, 1132, 1135, 1139, - 1143, 1147, 1151, 1154, 1157, 1161, 1165, 1169, - 1173, 1176, 1179, 1183, 1187, 1191, 1194, 1197, - 1201, 1205, 1209, 1213, 1217, 1220, 1223, 1228, - 1233, 1237, 1241, 1244, 1247, 1251, 1255, 1258, - 1261, 1265, 1269, 1273, 1276, 1279, 1283, 1287, - 1291, 1295, 1298, 1301, 1305, 1309, 1313, 1317, - 1320, 1323, 1327, 1331, 1335, 1339, 1342, 1345, - 1349, 1353, 1357, 1361, 1364, 1367, 1371, 1375, - 1379, 1383, 1386, 1389, 1394, 1398, 1402, 1406, - 1409, 1412, 1416, 1420, 1424, 1427, 1430, 1434, - 1438, 1442, 1446, 1449, 1452, 1456, 1460, 1464, - 1468, 1471, 1474, 1478, 1482, 1486, 1489, 1492, - 1496, 1500, 1504, 1508, 1512, 1515, 1518, 1521, - 1524, 1526, 1528, 1531, 1538, 1540, 1542, 1544, - 1546, 1548, 1550, 1577, 1579, 1581, 1583, 1585, - 1592, 1599, 1613, 1615, 1621, 1624, 1633, 1634, - 1637, 1640, 1647, 1649, 1651, 1653, 1656, 1701, - 1703, 1705, 1709, 1713, 1715, 1716, 1716, 1722, - 1724, 1726, 1728, 1730, 1733, 1734, 1735, 1742, - 1748, 1754, 1756, 1758, 1760, 1761, 1764, 1787, + 874, 875, 876, 877, 880, 883, 885, 900, + 903, 906, 908, 922, 927, 932, 936, 940, + 943, 946, 950, 954, 957, 960, 964, 968, + 972, 975, 978, 982, 986, 990, 994, 997, + 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, + 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, + 1060, 1063, 1066, 1070, 1074, 1078, 1082, 1085, + 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, + 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, + 1148, 1151, 1155, 1159, 1163, 1167, 1170, 1173, + 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, + 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, + 1238, 1241, 1245, 1249, 1252, 1255, 1259, 1263, + 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, + 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, + 1325, 1329, 1333, 1336, 1339, 1343, 1347, 1351, + 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, + 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, + 1414, 1418, 1421, 1424, 1428, 1432, 1436, 1440, + 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, + 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, + 1502, 1506, 1509, 1512, 1515, 1518, 1520, 1522, + 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, + 1546, 1548, 1550, 1577, 1579, 1586, 1593, 1607, + 1609, 1615, 1618, 1627, 1628, 1631, 1634, 1641, + 1643, 1645, 1647, 1650, 1695, 1697, 1699, 1703, + 1707, 1709, 1710, 1710, 1716, 1718, 1720, 1722, + 1724, 1727, 1728, 1729, 1736, 1742, 1748, 1750, + 1752, 1754, 1756, 1758, 1760, 1761, 1764, 1787, 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1906, 1908, 1910, - 1913, 1952, 1954, 1956, 1958, 1960, 1961, 1961, - 1962, 1963, 1970, 1976, 1982, 1985, 1987, 1998, - 2000, 2002, 2004, 2005, 2016, 2018, 2020, 2022, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { - -128, -65, -128, -65, -128, -65, 41, 33, - 35, 38, 39, 40, 41, 43, 45, 58, - 60, 61, 62, 63, 67, 80, 105, 109, - 115, 120, 123, 48, 57, 41, 95, 48, - 57, 65, 90, 97, 122, 39, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 39, 95, 48, 57, - 65, 90, 97, 122, 41, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 62, 95, 48, 57, - 65, 90, 97, 122, 33, 60, 61, 33, - 61, 38, 41, 95, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 48, 57, 41, 58, 105, 109, - 115, 120, 62, 95, 48, 57, 65, 90, - 97, 122, 41, 48, 57, 95, 48, 57, - 65, 90, 97, 122, 95, 48, 57, 65, - 90, 97, 122, 41, 95, 48, 57, 65, - 90, 97, 122, 95, 48, 57, 65, 90, - 97, 122, 105, 109, 115, 120, 41, 45, - 58, 105, 109, 115, 120, 46, 92, 93, - 46, 92, 93, 46, 92, 58, 92, 93, - 58, 92, 93, 58, 92, 61, 92, 93, - 61, 92, 93, 61, 92, 39, 48, 57, - 62, 45, 95, 48, 57, 65, 90, 97, - 122, 48, 57, 125, 48, 57, 125, 48, - 57, 125, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 39, 95, 48, 57, 65, 90, 97, - 122, 95, 48, 57, 65, 90, 97, 122, - 62, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 95, - 125, 48, 57, 65, 90, 97, 122, 48, - 55, 125, 48, 55, 125, 48, 57, 65, - 70, 97, 102, 44, 125, 48, 57, 125, - 48, 57, 125, 48, 57, 41, 41, 80, - 41, 41, 70, 41, 56, 41, 121, 97, - 109, 98, 105, 99, 101, 110, 105, 97, - 110, 101, 115, 116, 97, 110, 108, 109, - 116, 105, 110, 101, 115, 101, 117, 109, - 97, 107, 110, 103, 97, 108, 105, 112, - 111, 109, 111, 102, 111, 97, 104, 105, - 109, 105, 108, 108, 101, 103, 104, 105, - 110, 101, 115, 101, 105, 100, 110, 114, - 97, 100, 105, 97, 110, 95, 65, 98, - 111, 114, 105, 103, 105, 110, 97, 108, - 105, 97, 110, 97, 101, 109, 114, 111, - 107, 101, 101, 109, 111, 110, 116, 105, - 99, 110, 101, 105, 102, 111, 114, 109, - 112, 114, 114, 105, 111, 116, 105, 108, - 108, 105, 99, 115, 118, 101, 114, 101, - 116, 97, 110, 97, 103, 97, 114, 105, - 121, 112, 116, 105, 97, 110, 95, 72, - 105, 101, 114, 111, 103, 108, 121, 112, - 104, 115, 104, 105, 111, 112, 105, 99, - 111, 114, 103, 105, 97, 110, 97, 103, - 111, 108, 105, 116, 105, 99, 116, 104, - 105, 99, 101, 101, 107, 106, 114, 97, - 114, 97, 116, 105, 109, 117, 107, 104, - 105, 110, 117, 108, 110, 111, 111, 98, - 114, 101, 119, 114, 97, 103, 97, 110, - 97, 112, 101, 114, 105, 97, 108, 95, - 65, 114, 97, 109, 97, 105, 99, 104, - 115, 101, 114, 105, 116, 101, 100, 99, - 114, 105, 112, 116, 105, 111, 110, 97, - 108, 95, 80, 97, 104, 114, 108, 97, - 118, 105, 116, 104, 105, 97, 110, 118, - 97, 110, 101, 115, 101, 105, 110, 116, - 121, 116, 104, 105, 110, 97, 100, 97, - 97, 107, 97, 110, 97, 97, 104, 95, - 76, 105, 97, 109, 114, 111, 115, 104, - 116, 104, 105, 101, 114, 111, 116, 105, - 110, 112, 99, 104, 97, 109, 110, 115, - 98, 117, 101, 97, 114, 95, 66, 117, - 99, 100, 105, 97, 110, 105, 97, 110, - 108, 110, 97, 121, 97, 108, 97, 109, - 100, 97, 105, 99, 116, 101, 105, 95, - 77, 97, 121, 101, 107, 110, 103, 111, - 108, 105, 97, 110, 97, 110, 109, 97, - 114, 119, 95, 84, 97, 105, 95, 76, - 117, 101, 111, 104, 97, 109, 95, 100, - 67, 104, 105, 107, 105, 95, 73, 80, - 83, 84, 116, 97, 108, 105, 99, 101, - 114, 115, 105, 97, 110, 111, 117, 116, - 104, 95, 65, 114, 97, 98, 105, 97, - 110, 117, 114, 107, 105, 99, 105, 121, - 97, 109, 97, 110, 121, 97, 97, 111, - 103, 115, 95, 80, 97, 101, 110, 105, - 99, 105, 97, 110, 106, 97, 110, 103, - 110, 105, 99, 109, 117, 97, 114, 105, - 116, 97, 110, 114, 97, 115, 104, 116, - 114, 97, 97, 118, 105, 97, 110, 110, - 104, 97, 108, 97, 110, 100, 97, 110, - 101, 115, 101, 108, 114, 111, 116, 105, - 95, 78, 97, 103, 114, 105, 105, 97, - 99, 103, 105, 109, 97, 98, 108, 111, - 103, 97, 110, 119, 97, 95, 76, 84, - 86, 101, 104, 97, 109, 105, 101, 116, - 105, 108, 108, 117, 103, 117, 97, 97, - 105, 110, 97, 98, 102, 101, 116, 97, - 110, 105, 110, 97, 103, 104, 97, 114, - 105, 116, 105, 99, 105, 110, 115, 112, - 100, 123, 94, 125, 94, -128, -65, -128, - -65, -128, -65, 46, 92, 93, 46, 92, - 93, 46, 92, 58, 92, 93, 94, 97, - 98, 99, 100, 103, 108, 112, 115, 117, - 119, 120, 58, 92, 93, 58, 92, 93, - 58, 92, 58, 92, 93, 97, 98, 99, - 100, 103, 108, 112, 115, 117, 119, 120, - 58, 92, 93, 108, 115, 58, 92, 93, - 110, 112, 58, 92, 93, 117, 58, 92, - 93, 109, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 104, 58, 92, 93, 97, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 99, 58, 92, 93, 105, 58, 92, - 93, 105, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 108, 58, 92, 93, 97, - 58, 92, 93, 110, 58, 92, 93, 107, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 114, 58, 92, 93, 108, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 105, - 58, 92, 93, 103, 58, 92, 93, 105, - 58, 92, 93, 116, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 114, 58, 92, - 93, 97, 58, 92, 93, 112, 58, 92, - 93, 104, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 111, 58, 92, 93, 119, - 58, 92, 93, 101, 58, 92, 93, 114, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 114, 117, 58, 92, 93, 105, 58, - 92, 93, 110, 58, 92, 93, 116, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 99, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 112, 58, 92, 93, 97, 58, - 92, 93, 99, 58, 92, 93, 101, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 112, 58, 92, 93, 112, 58, 92, 93, - 101, 58, 92, 93, 114, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 111, 58, - 92, 93, 114, 58, 92, 93, 100, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 100, 58, 92, 93, 105, 58, 92, 93, - 103, 58, 92, 93, 105, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 115, 58, 92, 93, 110, - 112, 58, 92, 93, 117, 58, 92, 93, - 109, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 104, 58, 92, 93, 97, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 99, 58, 92, 93, 105, 58, 92, 93, - 105, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 58, 92, 93, 97, 58, - 92, 93, 110, 58, 92, 93, 107, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 116, 58, 92, 93, - 114, 58, 92, 93, 108, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 105, 58, - 92, 93, 103, 58, 92, 93, 105, 58, - 92, 93, 116, 58, 92, 93, 58, 92, - 93, 58, 92, 93, 114, 58, 92, 93, - 97, 58, 92, 93, 112, 58, 92, 93, - 104, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 111, 58, 92, 93, 119, 58, - 92, 93, 101, 58, 92, 93, 114, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 114, 117, 58, 92, 93, 105, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 110, - 58, 92, 93, 99, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 112, 58, 92, 93, 97, 58, 92, - 93, 99, 58, 92, 93, 101, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 112, - 58, 92, 93, 112, 58, 92, 93, 101, - 58, 92, 93, 114, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 111, 58, 92, - 93, 114, 58, 92, 93, 100, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 100, - 58, 92, 93, 105, 58, 92, 93, 103, - 58, 92, 93, 105, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 61, 92, - 93, 61, 92, 93, 61, 92, 48, 55, - 125, 48, 55, 125, 48, 57, 65, 70, - 97, 102, -128, -65, -128, -65, -128, -65, - -128, -65, -128, -65, -128, -65, 0, 32, - 35, 36, 40, 41, 42, 43, 46, 63, - 91, 92, 94, 123, 124, -128, -65, -64, - -33, -32, -17, -16, -9, -8, -1, 9, - 13, -128, -65, -128, -65, -128, -65, 42, - 63, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 39, - 48, 60, 63, 82, 95, 49, 55, 56, - 57, 65, 90, 97, 122, 48, 57, 105, - 109, 115, 120, 48, 57, 41, 48, 57, - 33, 61, 95, 48, 57, 65, 90, 97, - 122, 123, 41, 48, 57, 60, 61, 62, - 41, 45, 58, 105, 109, 115, 120, 43, - 63, 43, 63, 43, 63, 46, 58, 61, - 48, 65, 66, 67, 68, 69, 71, 72, - 75, 76, 78, 80, 81, 82, 83, 85, - 86, 87, 88, 90, 97, 98, 99, 100, - 101, 102, 103, 104, 107, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 122, 49, 55, 56, 57, 48, 55, 48, - 55, 48, 55, 56, 57, 48, 55, 56, - 57, 48, 57, 123, 39, 45, 60, 123, - 48, 57, 48, 57, 48, 57, 48, 57, - 48, 57, 39, 60, 123, 123, 123, 123, - 48, 57, 65, 70, 97, 102, 48, 57, - 65, 70, 97, 102, 48, 57, 65, 70, - 97, 102, 48, 57, 43, 63, 41, 85, - 41, 41, 67, 84, 65, 66, 67, 68, - 69, 71, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 82, 83, 84, 85, 86, - 88, 89, 90, 110, 114, 118, 97, 101, - 111, 114, 117, 97, 99, 102, 104, 110, - 111, 115, 117, 121, 109, 112, 101, 103, - 116, 101, 108, 111, 114, 117, 97, 101, - 105, 103, 117, 109, 110, 97, 97, 104, - 38, 97, 101, 105, 108, 109, 111, 116, - 117, 121, 97, 99, 101, 110, 111, 121, - 101, 100, 101, 107, 108, 111, 103, 108, - 114, 115, 99, 100, 101, 102, 104, 105, - 111, 115, 101, 117, 97, 99, 104, 105, - 107, 109, 111, 117, 121, 97, 101, 104, - 105, 103, 97, 97, 112, 115, 119, 105, - 108, 112, 115, 67, 76, 77, 78, 80, - 83, 90, 45, 91, 92, 93, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 46, 58, - 61, 48, 68, 69, 72, 76, 78, 80, - 81, 83, 85, 86, 87, 97, 98, 99, - 100, 101, 102, 103, 104, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 49, 55, 56, 57, 65, 90, 105, 122, - 48, 55, 48, 55, 48, 55, 48, 55, - 123, 123, 123, 123, 48, 57, 65, 70, - 97, 102, 48, 57, 65, 70, 97, 102, - 48, 57, 65, 70, 97, 102, 92, 93, - 94, 69, 81, 92, -128, -65, -64, -33, - -32, -17, -16, -9, -8, -1, -128, -65, - -128, -65, -128, -65, 69, 92, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 69, 41, - 10, 0 + static const unsigned char _regex_trans_keys[] = { + 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, + 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, + 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 39u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 39u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 41u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 62u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 33u, 60u, + 61u, 33u, 61u, 38u, 41u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 48u, 57u, 41u, 58u, + 105u, 109u, 115u, 120u, 62u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 41u, 48u, 57u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 41u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 105u, 109u, 115u, 120u, + 41u, 45u, 58u, 105u, 109u, 115u, 120u, 46u, + 92u, 93u, 46u, 92u, 93u, 46u, 92u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 61u, + 92u, 93u, 61u, 92u, 93u, 61u, 92u, 39u, + 48u, 57u, 62u, 45u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 48u, 57u, 125u, 48u, 57u, + 125u, 48u, 57u, 125u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 39u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 62u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 95u, 125u, 48u, 57u, 65u, 90u, 97u, + 122u, 48u, 55u, 125u, 48u, 55u, 125u, 48u, + 57u, 65u, 70u, 97u, 102u, 44u, 125u, 48u, + 57u, 125u, 48u, 57u, 125u, 48u, 57u, 128u, + 191u, 128u, 191u, 128u, 191u, 41u, 41u, 80u, + 41u, 41u, 70u, 41u, 56u, 41u, 121u, 97u, + 109u, 98u, 105u, 99u, 101u, 110u, 105u, 97u, + 110u, 101u, 115u, 116u, 97u, 110u, 108u, 109u, + 116u, 105u, 110u, 101u, 115u, 101u, 117u, 109u, + 97u, 107u, 110u, 103u, 97u, 108u, 105u, 112u, + 111u, 109u, 111u, 102u, 111u, 97u, 104u, 105u, + 109u, 105u, 108u, 108u, 101u, 103u, 104u, 105u, + 110u, 101u, 115u, 101u, 105u, 100u, 110u, 114u, + 97u, 100u, 105u, 97u, 110u, 95u, 65u, 98u, + 111u, 114u, 105u, 103u, 105u, 110u, 97u, 108u, + 105u, 97u, 110u, 97u, 101u, 109u, 114u, 111u, + 107u, 101u, 101u, 109u, 111u, 110u, 116u, 105u, + 99u, 110u, 101u, 105u, 102u, 111u, 114u, 109u, + 112u, 114u, 114u, 105u, 111u, 116u, 105u, 108u, + 108u, 105u, 99u, 115u, 118u, 101u, 114u, 101u, + 116u, 97u, 110u, 97u, 103u, 97u, 114u, 105u, + 121u, 112u, 116u, 105u, 97u, 110u, 95u, 72u, + 105u, 101u, 114u, 111u, 103u, 108u, 121u, 112u, + 104u, 115u, 104u, 105u, 111u, 112u, 105u, 99u, + 111u, 114u, 103u, 105u, 97u, 110u, 97u, 103u, + 111u, 108u, 105u, 116u, 105u, 99u, 116u, 104u, + 105u, 99u, 101u, 101u, 107u, 106u, 114u, 97u, + 114u, 97u, 116u, 105u, 109u, 117u, 107u, 104u, + 105u, 110u, 117u, 108u, 110u, 111u, 111u, 98u, + 114u, 101u, 119u, 114u, 97u, 103u, 97u, 110u, + 97u, 112u, 101u, 114u, 105u, 97u, 108u, 95u, + 65u, 114u, 97u, 109u, 97u, 105u, 99u, 104u, + 115u, 101u, 114u, 105u, 116u, 101u, 100u, 99u, + 114u, 105u, 112u, 116u, 105u, 111u, 110u, 97u, + 108u, 95u, 80u, 97u, 104u, 114u, 108u, 97u, + 118u, 105u, 116u, 104u, 105u, 97u, 110u, 118u, + 97u, 110u, 101u, 115u, 101u, 105u, 110u, 116u, + 121u, 116u, 104u, 105u, 110u, 97u, 100u, 97u, + 97u, 107u, 97u, 110u, 97u, 97u, 104u, 95u, + 76u, 105u, 97u, 109u, 114u, 111u, 115u, 104u, + 116u, 104u, 105u, 101u, 114u, 111u, 116u, 105u, + 110u, 112u, 99u, 104u, 97u, 109u, 110u, 115u, + 98u, 117u, 101u, 97u, 114u, 95u, 66u, 117u, + 99u, 100u, 105u, 97u, 110u, 105u, 97u, 110u, + 108u, 110u, 97u, 121u, 97u, 108u, 97u, 109u, + 100u, 97u, 105u, 99u, 116u, 101u, 105u, 95u, + 77u, 97u, 121u, 101u, 107u, 110u, 103u, 111u, + 108u, 105u, 97u, 110u, 97u, 110u, 109u, 97u, + 114u, 119u, 95u, 84u, 97u, 105u, 95u, 76u, + 117u, 101u, 111u, 104u, 97u, 109u, 95u, 100u, + 67u, 104u, 105u, 107u, 105u, 95u, 73u, 80u, + 83u, 84u, 116u, 97u, 108u, 105u, 99u, 101u, + 114u, 115u, 105u, 97u, 110u, 111u, 117u, 116u, + 104u, 95u, 65u, 114u, 97u, 98u, 105u, 97u, + 110u, 117u, 114u, 107u, 105u, 99u, 105u, 121u, + 97u, 109u, 97u, 110u, 121u, 97u, 97u, 111u, + 103u, 115u, 95u, 80u, 97u, 101u, 110u, 105u, + 99u, 105u, 97u, 110u, 106u, 97u, 110u, 103u, + 110u, 105u, 99u, 109u, 117u, 97u, 114u, 105u, + 116u, 97u, 110u, 114u, 97u, 115u, 104u, 116u, + 114u, 97u, 97u, 118u, 105u, 97u, 110u, 110u, + 104u, 97u, 108u, 97u, 110u, 100u, 97u, 110u, + 101u, 115u, 101u, 108u, 114u, 111u, 116u, 105u, + 95u, 78u, 97u, 103u, 114u, 105u, 105u, 97u, + 99u, 103u, 105u, 109u, 97u, 98u, 108u, 111u, + 103u, 97u, 110u, 119u, 97u, 95u, 76u, 84u, + 86u, 101u, 104u, 97u, 109u, 105u, 101u, 116u, + 105u, 108u, 108u, 117u, 103u, 117u, 97u, 97u, + 105u, 110u, 97u, 98u, 102u, 101u, 116u, 97u, + 110u, 105u, 110u, 97u, 103u, 104u, 97u, 114u, + 105u, 116u, 105u, 99u, 105u, 110u, 115u, 112u, + 100u, 123u, 94u, 125u, 94u, 46u, 92u, 93u, + 46u, 92u, 93u, 46u, 92u, 58u, 92u, 93u, + 94u, 97u, 98u, 99u, 100u, 103u, 108u, 112u, + 115u, 117u, 119u, 120u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 58u, 92u, 93u, 97u, + 98u, 99u, 100u, 103u, 108u, 112u, 115u, 117u, + 119u, 120u, 58u, 92u, 93u, 108u, 115u, 58u, + 92u, 93u, 110u, 112u, 58u, 92u, 93u, 117u, + 58u, 92u, 93u, 109u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 104u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 105u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 108u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 110u, 58u, 92u, + 93u, 107u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 108u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 103u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 116u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 114u, + 58u, 92u, 93u, 97u, 58u, 92u, 93u, 112u, + 58u, 92u, 93u, 104u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 111u, 58u, 92u, + 93u, 119u, 58u, 92u, 93u, 101u, 58u, 92u, + 93u, 114u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 114u, 117u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 116u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 99u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 112u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 99u, 58u, 92u, 93u, + 101u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 112u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 101u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 111u, 58u, 92u, 93u, 114u, 58u, 92u, 93u, + 100u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 100u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 103u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 115u, 58u, 92u, + 93u, 110u, 112u, 58u, 92u, 93u, 117u, 58u, + 92u, 93u, 109u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 104u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 99u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 105u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 107u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 116u, 58u, + 92u, 93u, 114u, 58u, 92u, 93u, 108u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 103u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 116u, 58u, 92u, 93u, + 58u, 92u, 93u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 97u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 104u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 111u, 58u, 92u, 93u, + 119u, 58u, 92u, 93u, 101u, 58u, 92u, 93u, + 114u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 114u, 117u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 110u, 58u, 92u, 93u, 99u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 112u, 58u, 92u, 93u, 97u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 101u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 112u, 58u, 92u, 93u, 112u, 58u, 92u, + 93u, 101u, 58u, 92u, 93u, 114u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 111u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 100u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 100u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 103u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 61u, 92u, 93u, 61u, 92u, 93u, 61u, 92u, + 48u, 55u, 125u, 48u, 55u, 125u, 48u, 57u, + 65u, 70u, 97u, 102u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 0u, 32u, + 35u, 36u, 40u, 41u, 42u, 43u, 46u, 63u, + 91u, 92u, 94u, 123u, 124u, 9u, 13u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 42u, 63u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 39u, 48u, 60u, 63u, 82u, 95u, 49u, + 55u, 56u, 57u, 65u, 90u, 97u, 122u, 48u, + 57u, 105u, 109u, 115u, 120u, 48u, 57u, 41u, + 48u, 57u, 33u, 61u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 123u, 41u, 48u, 57u, 60u, + 61u, 62u, 41u, 45u, 58u, 105u, 109u, 115u, + 120u, 43u, 63u, 43u, 63u, 43u, 63u, 46u, + 58u, 61u, 48u, 65u, 66u, 67u, 68u, 69u, + 71u, 72u, 75u, 76u, 78u, 80u, 81u, 82u, + 83u, 85u, 86u, 87u, 88u, 90u, 97u, 98u, + 99u, 100u, 101u, 102u, 103u, 104u, 107u, 108u, + 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, + 119u, 120u, 122u, 49u, 55u, 56u, 57u, 48u, + 55u, 48u, 55u, 48u, 55u, 56u, 57u, 48u, + 55u, 56u, 57u, 48u, 57u, 123u, 39u, 45u, + 60u, 123u, 48u, 57u, 48u, 57u, 48u, 57u, + 48u, 57u, 48u, 57u, 39u, 60u, 123u, 123u, + 123u, 123u, 48u, 57u, 65u, 70u, 97u, 102u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 43u, 63u, + 128u, 191u, 128u, 191u, 128u, 191u, 41u, 85u, + 41u, 41u, 67u, 84u, 65u, 66u, 67u, 68u, + 69u, 71u, 72u, 73u, 74u, 75u, 76u, 77u, + 78u, 79u, 80u, 82u, 83u, 84u, 85u, 86u, + 88u, 89u, 90u, 110u, 114u, 118u, 97u, 101u, + 111u, 114u, 117u, 97u, 99u, 102u, 104u, 110u, + 111u, 115u, 117u, 121u, 109u, 112u, 101u, 103u, + 116u, 101u, 108u, 111u, 114u, 117u, 97u, 101u, + 105u, 103u, 117u, 109u, 110u, 97u, 97u, 104u, + 38u, 97u, 101u, 105u, 108u, 109u, 111u, 116u, + 117u, 121u, 97u, 99u, 101u, 110u, 111u, 121u, + 101u, 100u, 101u, 107u, 108u, 111u, 103u, 108u, + 114u, 115u, 99u, 100u, 101u, 102u, 104u, 105u, + 111u, 115u, 101u, 117u, 97u, 99u, 104u, 105u, + 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, + 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, + 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, + 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { - 0, 0, 0, 0, 1, 20, 2, 2, - 1, 2, 1, 2, 2, 2, 1, 2, - 3, 2, 3, 1, 2, 2, 1, 6, - 2, 1, 1, 1, 2, 1, 4, 7, - 3, 3, 2, 3, 3, 2, 3, 3, - 2, 1, 0, 1, 2, 0, 1, 1, - 1, 2, 2, 2, 2, 1, 2, 1, - 2, 1, 2, 0, 1, 1, 2, 1, - 1, 1, 2, 1, 2, 2, 1, 1, + 0, 1, 20, 2, 2, 1, 2, 1, + 2, 2, 2, 1, 2, 3, 2, 3, + 1, 2, 2, 1, 6, 2, 1, 1, + 1, 2, 1, 4, 7, 3, 3, 2, + 3, 3, 2, 3, 3, 2, 1, 0, + 1, 2, 0, 1, 1, 1, 2, 2, + 2, 2, 1, 2, 1, 2, 1, 2, + 0, 1, 1, 2, 1, 1, 0, 0, + 0, 1, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -835,54 +836,54 @@ static const short _regex_actions[] = { 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 2, 15, 3, 3, 2, 14, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 1, 1, 1, 3, 3, 2, 15, 3, + 3, 2, 14, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 4, 3, 3, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 3, 3, - 2, 0, 1, 1, 0, 0, 0, 0, - 0, 0, 15, 0, 0, 0, 2, 1, - 1, 6, 0, 4, 1, 3, 1, 1, - 3, 7, 2, 2, 2, 3, 41, 0, - 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 3, 1, 1, 1, 0, - 0, 0, 2, 2, 1, 3, 23, 3, + 4, 3, 3, 3, 3, 2, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 15, 2, 1, 1, 6, 0, + 4, 1, 3, 1, 1, 3, 7, 2, + 2, 2, 3, 41, 0, 0, 0, 0, + 0, 1, 0, 4, 0, 0, 0, 0, + 3, 1, 1, 1, 0, 0, 0, 2, + 0, 0, 0, 2, 1, 3, 23, 3, 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 0, 0, 0, 3, - 31, 0, 0, 0, 0, 1, 0, 1, - 1, 1, 0, 0, 3, 2, 1, 0, - 0, 0, 1, 1, 0, 0, 0, 1, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { - 0, 1, 1, 1, 0, 1, 3, 3, - 3, 3, 0, 3, 3, 3, 3, 3, - 0, 0, 3, 3, 3, 3, 1, 0, - 3, 1, 3, 3, 3, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 3, 1, 1, 1, - 0, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 1, 1, 3, 1, 1, + 0, 0, 1, 3, 3, 3, 3, 0, + 3, 3, 3, 3, 3, 0, 0, 3, + 3, 3, 3, 1, 0, 3, 1, 3, + 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 3, 1, 1, 1, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 3, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -945,7 +946,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -967,33 +967,34 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 3, 1, 1, 1, 1, - 1, 1, 6, 1, 1, 1, 0, 3, - 3, 4, 1, 1, 1, 3, 0, 1, - 0, 0, 0, 0, 0, 0, 2, 1, - 1, 2, 2, 1, 0, 0, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 3, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 6, 0, 3, 3, 4, 1, + 1, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 2, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 1, 0, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 1, 1, 1, 0, - 4, 1, 1, 1, 1, 0, 0, 0, - 0, 3, 3, 3, 0, 0, 5, 1, - 1, 1, 0, 5, 1, 1, 1, 0, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { - 0, 0, 2, 4, 6, 8, 30, 36, - 42, 47, 53, 55, 61, 67, 73, 78, - 84, 88, 91, 98, 103, 109, 115, 118, - 125, 131, 134, 139, 144, 150, 155, 160, - 168, 172, 176, 179, 183, 187, 190, 194, - 198, 201, 203, 205, 207, 213, 215, 218, - 221, 223, 229, 235, 241, 247, 252, 258, - 263, 269, 274, 280, 282, 285, 290, 294, - 297, 300, 302, 305, 307, 310, 313, 315, + 0, 0, 2, 24, 30, 36, 41, 47, + 49, 55, 61, 67, 72, 78, 82, 85, + 92, 97, 103, 109, 112, 119, 125, 128, + 133, 138, 144, 149, 154, 162, 166, 170, + 173, 177, 181, 184, 188, 192, 195, 197, + 199, 201, 207, 209, 212, 215, 217, 223, + 229, 235, 241, 246, 252, 257, 263, 268, + 274, 276, 279, 284, 288, 291, 294, 296, + 298, 300, 302, 305, 307, 310, 313, 315, 317, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 350, 352, 354, 356, 358, 360, 362, 364, 366, @@ -1055,47 +1056,46 @@ static const short _regex_actions[] = { 1279, 1281, 1283, 1286, 1288, 1290, 1292, 1294, 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, - 1328, 1330, 1332, 1334, 1336, 1338, 1340, 1344, - 1348, 1351, 1367, 1371, 1375, 1378, 1393, 1399, - 1405, 1410, 1415, 1419, 1423, 1428, 1433, 1437, - 1441, 1446, 1451, 1456, 1460, 1464, 1469, 1474, - 1479, 1484, 1488, 1492, 1497, 1502, 1507, 1512, - 1516, 1520, 1525, 1530, 1535, 1540, 1544, 1548, - 1553, 1558, 1563, 1568, 1572, 1576, 1581, 1586, - 1591, 1596, 1600, 1604, 1610, 1615, 1620, 1625, - 1629, 1633, 1638, 1643, 1648, 1652, 1656, 1661, - 1666, 1671, 1676, 1680, 1684, 1689, 1694, 1699, - 1704, 1708, 1712, 1717, 1722, 1727, 1731, 1735, - 1740, 1745, 1750, 1755, 1760, 1764, 1768, 1774, - 1780, 1785, 1790, 1794, 1798, 1803, 1808, 1812, - 1816, 1821, 1826, 1831, 1835, 1839, 1844, 1849, - 1854, 1859, 1863, 1867, 1872, 1877, 1882, 1887, - 1891, 1895, 1900, 1905, 1910, 1915, 1919, 1923, - 1928, 1933, 1938, 1943, 1947, 1951, 1956, 1961, - 1966, 1971, 1975, 1979, 1985, 1990, 1995, 2000, - 2004, 2008, 2013, 2018, 2023, 2027, 2031, 2036, - 2041, 2046, 2051, 2055, 2059, 2064, 2069, 2074, - 2079, 2083, 2087, 2092, 2097, 2102, 2106, 2110, - 2115, 2120, 2125, 2130, 2135, 2139, 2143, 2147, - 2151, 2154, 2156, 2159, 2164, 2166, 2168, 2170, - 2172, 2174, 2176, 2198, 2200, 2202, 2204, 2207, - 2212, 2217, 2228, 2230, 2236, 2239, 2246, 2248, - 2251, 2255, 2263, 2266, 2269, 2272, 2276, 2320, - 2322, 2324, 2327, 2330, 2332, 2334, 2335, 2341, - 2343, 2345, 2347, 2349, 2353, 2355, 2357, 2362, - 2366, 2370, 2372, 2375, 2378, 2380, 2384, 2408, + 1328, 1330, 1332, 1334, 1338, 1342, 1345, 1361, + 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, + 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, + 1450, 1454, 1458, 1463, 1468, 1473, 1478, 1482, + 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, + 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, + 1562, 1566, 1570, 1575, 1580, 1585, 1590, 1594, + 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, + 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, + 1674, 1678, 1683, 1688, 1693, 1698, 1702, 1706, + 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, + 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, + 1788, 1792, 1797, 1802, 1806, 1810, 1815, 1820, + 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, + 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, + 1899, 1904, 1909, 1913, 1917, 1922, 1927, 1932, + 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, + 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, + 2012, 2017, 2021, 2025, 2030, 2035, 2040, 2045, + 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, + 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, + 2124, 2129, 2133, 2137, 2141, 2145, 2148, 2150, + 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, + 2172, 2174, 2176, 2198, 2201, 2206, 2211, 2222, + 2224, 2230, 2233, 2240, 2242, 2245, 2249, 2257, + 2260, 2263, 2266, 2270, 2314, 2316, 2318, 2321, + 2324, 2326, 2328, 2329, 2335, 2337, 2339, 2341, + 2343, 2347, 2349, 2351, 2356, 2360, 2364, 2366, + 2369, 2371, 2373, 2375, 2378, 2380, 2384, 2408, 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2551, 2553, 2555, - 2559, 2595, 2597, 2599, 2601, 2603, 2605, 2606, - 2608, 2610, 2615, 2619, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { - 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1132,6 +1132,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1261,8 +1263,6 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1365,13 +1365,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 0, -1, 0, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1393,6 +1392,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1412,24 +1412,24 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, 0, 0, - 0, 0, 0, -1, 0, -1, 0, -1, - 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,48 +1534,49 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { - 0, 2, 3, 5, 6, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, - 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, 133, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 146, - 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, - 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 296, + 297, 299, 300, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, @@ -1704,137 +1705,137 @@ static const short _regex_actions[] = { 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, - 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1339, - 1340, 1342, 1343, 1345, 1346, 1347, 1348, 1349, - 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, - 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, - 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, - 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, - 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, - 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, - 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, - 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, - 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, - 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, - 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, - 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, - 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, - 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, - 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, - 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, - 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, - 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, - 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, - 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, - 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, - 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, - 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, - 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, - 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, - 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, - 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, - 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, - 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, - 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, - 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, - 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, - 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, - 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, - 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, - 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, - 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, - 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, - 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, - 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, - 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, - 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, - 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, - 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, - 1710, 1711, 1712, 1713, 1714, 1715, 1716, 1717, - 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, - 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, - 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, - 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, - 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, - 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, - 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, - 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, - 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, - 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, - 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, - 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, - 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, - 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, - 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, - 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, - 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, - 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, - 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, - 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, - 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, - 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, - 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, - 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, - 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, - 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, - 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, - 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, - 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, - 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, - 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, - 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, - 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, - 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, - 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, - 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, - 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, - 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, - 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, - 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, - 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, - 2046, 2047, 2048, 2049, 2050, 2051, 2052, 2053, - 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, - 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, - 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, - 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, - 2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, - 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, - 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, - 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, - 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, - 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, - 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, - 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, - 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, - 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, - 2166, 2167, 2168, 2169, 2170, 2172, 2173, 2175, + 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, + 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, + 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, + 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, + 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, + 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, + 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, + 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, + 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, + 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, + 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, + 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, + 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, + 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, + 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, + 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, + 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, + 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, + 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, + 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, + 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, + 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, + 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, + 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, + 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, + 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, + 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, + 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, + 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, + 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, + 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, + 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, + 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, + 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1602, + 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, + 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, + 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, + 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, + 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, + 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, + 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, + 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, + 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, + 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, + 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, + 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, + 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1714, + 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, + 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, + 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, + 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, + 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, + 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, + 1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, + 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, + 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, + 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, + 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, + 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, + 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, + 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, + 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, + 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842, + 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, + 1851, 1852, 1853, 1854, 1855, 1856, 1857, 1858, + 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, + 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, + 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, + 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, + 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, + 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, + 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, + 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, + 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, + 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, + 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, + 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, + 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, + 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, + 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, + 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, + 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, + 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, + 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, + 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, + 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, + 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, + 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, + 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, + 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, + 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, + 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, + 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, + 2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, + 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, + 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, + 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, + 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, + 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, + 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, + 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, + 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2163, + 2164, 2166, 2167, 2169, 2170, 2172, 2173, 2175, 2176, 2178, 2179, 2181, 2182, 2184, 2185, 2187, 2188, 2189, 2190, 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, 2204, - 2206, 2208, 2210, 2212, 2214, 2215, 2216, 2218, - 2219, 2221, 2222, 2224, 2225, 2226, 2227, 2228, - 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, - 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, - 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, - 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, - 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, - 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, - 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, - 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, - 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, - 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, - 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, - 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, - 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, - 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, - 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, - 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, - 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, - 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, - 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, - 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, - 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, + 2205, 2207, 2209, 2211, 2213, 2215, 2216, 2217, + 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, + 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, + 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, + 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, + 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, + 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, + 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, + 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, + 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, + 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, + 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, + 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, + 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, + 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, + 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, + 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, + 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, + 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, + 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, + 2386, 2387, 2389, 2390, 2392, 2393, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, @@ -1856,21 +1857,21 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2577, 2578, - 2580, 2581, 2583, 2584, 2585, 2586, 2587, 2588, - 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, - 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, - 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, - 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, - 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, - 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, - 2637, 2638, 2639, 2640, 2641, 2642, 2643, 2644, - 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2675, 2676, - 2678, 2679, 2681, 2682, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2698, 2699, 2701, - 2702, 2704, 2705, 2706, 2707, 2708, 2709, 2710, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, + 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, + 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, + 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, + 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, + 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, + 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, + 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, + 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,11 +1978,10 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { - 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2018,6 +2018,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2147,8 +2149,6 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2251,13 +2251,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2279,6 +2278,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2298,22 +2298,23 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 2, 2, - 2, 2, 2, 1, 2, 1, 2, 1, - 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,11 +2421,10 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { - 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2461,6 +2461,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2591,8 +2593,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2695,14 +2695,13 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 1, 0, 0, + 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2724,6 +2723,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 1, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2744,9 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 1, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2755,14 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,48 +2870,48 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { - 746, 746, 746, 746, 3, 746, 746, 746, - 746, 746, 746, 746, 746, 751, 752, 753, - 746, 754, 755, 746, 757, 746, 746, 758, - 759, 760, 761, 761, 761, 761, 746, 756, - 746, 746, 6, 6, 6, 6, 746, 746, - 7, 7, 7, 7, 746, 9, 9, 9, - 9, 746, 10, 9, 9, 9, 9, 746, - 746, 746, 746, 11, 11, 11, 11, 746, - 746, 11, 12, 11, 11, 746, 746, 11, - 12, 11, 11, 746, 15, 15, 15, 15, - 746, 10, 15, 15, 15, 15, 746, 746, - 17, 746, 746, 746, 746, 746, 19, 746, - 11, 21, 11, 11, 746, 20, 20, 20, - 20, 746, 746, 20, 20, 20, 20, 746, - 746, 11, 21, 11, 11, 746, 746, 22, - 746, 746, 746, 23, 23, 23, 23, 746, - 746, 24, 24, 24, 24, 746, 746, 25, - 746, 24, 24, 24, 24, 746, 28, 28, - 28, 28, 746, 746, 28, 28, 28, 28, - 746, 6, 6, 6, 6, 746, 23, 23, - 23, 23, 746, 746, 30, 746, 31, 31, - 31, 31, 746, 33, 34, 746, 32, 33, - 34, 746, 32, 33, 34, 32, 36, 37, - 746, 35, 36, 37, 746, 35, 36, 37, - 35, 39, 40, 746, 38, 39, 40, 746, - 38, 39, 40, 38, 746, 41, 775, 746, - 746, 43, 45, 52, 49, 52, 52, 746, - 46, 746, 746, 47, 746, 746, 48, 746, - 746, 746, 52, 746, 50, 52, 52, 746, - 52, 746, 51, 52, 52, 746, 52, 746, - 52, 52, 52, 746, 52, 746, 52, 52, - 52, 746, 54, 54, 54, 54, 746, 746, - 54, 54, 54, 54, 746, 56, 56, 56, - 56, 746, 746, 56, 56, 56, 56, 746, - 58, 58, 58, 58, 746, 58, 746, 58, - 58, 58, 746, 60, 746, 746, 60, 746, - 746, 61, 61, 61, 746, 63, 786, 62, - 746, 786, 64, 746, 786, 64, 746, 787, + 746, 746, 746, 746, 748, 749, 750, 746, + 751, 752, 746, 754, 746, 746, 755, 756, + 757, 758, 758, 758, 758, 746, 753, 746, + 746, 3, 3, 3, 3, 746, 746, 4, + 4, 4, 4, 746, 6, 6, 6, 6, + 746, 7, 6, 6, 6, 6, 746, 746, + 746, 746, 8, 8, 8, 8, 746, 746, + 8, 9, 8, 8, 746, 746, 8, 9, + 8, 8, 746, 12, 12, 12, 12, 746, + 7, 12, 12, 12, 12, 746, 746, 14, + 746, 746, 746, 746, 746, 16, 746, 8, + 18, 8, 8, 746, 17, 17, 17, 17, + 746, 746, 17, 17, 17, 17, 746, 746, + 8, 18, 8, 8, 746, 746, 19, 746, + 746, 746, 20, 20, 20, 20, 746, 746, + 21, 21, 21, 21, 746, 746, 22, 746, + 21, 21, 21, 21, 746, 25, 25, 25, + 25, 746, 746, 25, 25, 25, 25, 746, + 3, 3, 3, 3, 746, 20, 20, 20, + 20, 746, 746, 27, 746, 28, 28, 28, + 28, 746, 30, 31, 746, 29, 30, 31, + 746, 29, 30, 31, 29, 33, 34, 746, + 32, 33, 34, 746, 32, 33, 34, 32, + 36, 37, 746, 35, 36, 37, 746, 35, + 36, 37, 35, 746, 38, 772, 746, 746, + 40, 42, 49, 46, 49, 49, 746, 43, + 746, 746, 44, 746, 746, 45, 746, 746, + 746, 49, 746, 47, 49, 49, 746, 49, + 746, 48, 49, 49, 746, 49, 746, 49, + 49, 49, 746, 49, 746, 49, 49, 49, + 746, 51, 51, 51, 51, 746, 746, 51, + 51, 51, 51, 746, 53, 53, 53, 53, + 746, 746, 53, 53, 53, 53, 746, 55, + 55, 55, 55, 746, 55, 746, 55, 55, + 55, 746, 57, 746, 746, 57, 746, 746, + 58, 58, 58, 746, 60, 783, 59, 746, + 783, 61, 746, 783, 61, 746, 746, 746, + 746, 746, 64, 746, 746, 746, 746, 787, 65, 787, 67, 65, 787, 65, 787, 69, 65, 787, 70, 65, 787, 65, 790, 790, 73, 76, 790, 74, 790, 75, 790, 790, @@ -3040,139 +3041,139 @@ static const short _regex_actions[] = { 790, 552, 790, 553, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 560, 0, 562, 561, 817, 0, 0, - 561, 819, 819, 819, 819, 565, 819, 819, - 819, 819, 567, 568, 819, 566, 567, 568, - 819, 566, 567, 568, 566, 571, 572, 819, - 573, 654, 669, 675, 681, 687, 693, 699, - 710, 716, 722, 727, 570, 571, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 570, - 571, 572, 819, 574, 589, 595, 601, 607, - 613, 619, 630, 636, 642, 647, 570, 571, - 572, 819, 575, 584, 570, 571, 572, 819, - 576, 580, 570, 571, 572, 819, 577, 570, - 571, 572, 819, 578, 570, 579, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 581, 570, 571, 572, 819, 582, 570, 583, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 585, 570, 571, 572, 819, 586, - 570, 571, 572, 819, 587, 570, 588, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 590, 570, 571, 572, 819, 591, 570, - 571, 572, 819, 592, 570, 571, 572, 819, - 593, 570, 594, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 596, 570, 571, - 572, 819, 597, 570, 571, 572, 819, 598, - 570, 571, 572, 819, 599, 570, 600, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 602, 570, 571, 572, 819, 603, 570, - 571, 572, 819, 604, 570, 571, 572, 819, - 605, 570, 606, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 608, 570, 571, - 572, 819, 609, 570, 571, 572, 819, 610, - 570, 571, 572, 819, 611, 570, 612, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 614, 570, 571, 572, 819, 615, 570, - 571, 572, 819, 616, 570, 571, 572, 819, - 617, 570, 618, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 620, 625, 570, - 571, 572, 819, 621, 570, 571, 572, 819, - 622, 570, 571, 572, 819, 623, 570, 624, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 626, 570, 571, 572, 819, 627, - 570, 571, 572, 819, 628, 570, 629, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 631, 570, 571, 572, 819, 632, 570, - 571, 572, 819, 633, 570, 571, 572, 819, - 634, 570, 635, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 637, 570, 571, - 572, 819, 638, 570, 571, 572, 819, 639, - 570, 571, 572, 819, 640, 570, 641, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 643, 570, 571, 572, 819, 644, 570, - 571, 572, 819, 645, 570, 646, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 648, 570, 571, 572, 819, 649, 570, 571, - 572, 819, 650, 570, 571, 572, 819, 651, - 570, 571, 572, 819, 652, 570, 653, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 655, 664, 570, 571, 572, 819, 656, - 660, 570, 571, 572, 819, 657, 570, 571, - 572, 819, 658, 570, 659, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 661, - 570, 571, 572, 819, 662, 570, 663, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 665, 570, 571, 572, 819, 666, 570, - 571, 572, 819, 667, 570, 668, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 670, 570, 571, 572, 819, 671, 570, 571, - 572, 819, 672, 570, 571, 572, 819, 673, - 570, 674, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 676, 570, 571, 572, - 819, 677, 570, 571, 572, 819, 678, 570, - 571, 572, 819, 679, 570, 680, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 682, 570, 571, 572, 819, 683, 570, 571, - 572, 819, 684, 570, 571, 572, 819, 685, - 570, 686, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 688, 570, 571, 572, - 819, 689, 570, 571, 572, 819, 690, 570, - 571, 572, 819, 691, 570, 692, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 694, 570, 571, 572, 819, 695, 570, 571, - 572, 819, 696, 570, 571, 572, 819, 697, - 570, 698, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 700, 705, 570, 571, - 572, 819, 701, 570, 571, 572, 819, 702, - 570, 571, 572, 819, 703, 570, 704, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 706, 570, 571, 572, 819, 707, 570, - 571, 572, 819, 708, 570, 709, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 711, 570, 571, 572, 819, 712, 570, 571, - 572, 819, 713, 570, 571, 572, 819, 714, - 570, 715, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 717, 570, 571, 572, - 819, 718, 570, 571, 572, 819, 719, 570, - 571, 572, 819, 720, 570, 721, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 723, 570, 571, 572, 819, 724, 570, 571, - 572, 819, 725, 570, 726, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 728, - 570, 571, 572, 819, 729, 570, 571, 572, - 819, 730, 570, 571, 572, 819, 731, 570, - 571, 572, 819, 732, 570, 733, 572, 819, - 570, 571, 572, 819, 570, 735, 736, 819, - 734, 735, 736, 819, 734, 735, 736, 734, - 738, 819, 819, 738, 819, 819, 739, 739, - 739, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, - 746, 750, 746, 762, 763, 746, 764, 765, - 766, 746, 785, 746, 746, 746, 746, 747, - 746, 748, 746, 749, 746, 746, 746, 746, - 746, 746, 746, 746, 1, 746, 746, 2, - 746, 4, 5, 746, 6, 6, 6, 6, - 746, 7, 7, 7, 7, 746, 8, 11, - 14, 16, 18, 11, 12, 13, 11, 11, - 746, 22, 746, 23, 23, 23, 23, 22, - 746, 746, 22, 746, 746, 746, 24, 24, - 24, 24, 746, 746, 746, 746, 25, 746, - 26, 27, 29, 746, 746, 30, 746, 31, - 31, 31, 31, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 32, 35, 38, - 746, 767, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 772, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 773, - 746, 746, 746, 774, 746, 779, 746, 746, - 780, 781, 746, 746, 746, 746, 746, 746, - 782, 746, 769, 771, 746, 768, 746, 746, - 746, 770, 771, 746, 771, 771, 746, 771, - 746, 746, 746, 746, 41, 42, 43, 44, - 777, 746, 776, 746, 746, 746, 778, 746, - 746, 746, 53, 55, 57, 746, 59, 746, - 746, 746, 784, 783, 783, 783, 746, 746, - 746, 746, 746, 61, 61, 61, 746, 62, - 746, 746, 746, 746, 787, 789, 788, 787, + 561, 564, 565, 819, 563, 564, 565, 819, + 563, 564, 565, 563, 568, 569, 819, 570, + 651, 666, 672, 678, 684, 690, 696, 707, + 713, 719, 724, 567, 568, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 567, 568, + 569, 819, 571, 586, 592, 598, 604, 610, + 616, 627, 633, 639, 644, 567, 568, 569, + 819, 572, 581, 567, 568, 569, 819, 573, + 577, 567, 568, 569, 819, 574, 567, 568, + 569, 819, 575, 567, 576, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 578, + 567, 568, 569, 819, 579, 567, 580, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 582, 567, 568, 569, 819, 583, 567, + 568, 569, 819, 584, 567, 585, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 587, 567, 568, 569, 819, 588, 567, 568, + 569, 819, 589, 567, 568, 569, 819, 590, + 567, 591, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 593, 567, 568, 569, + 819, 594, 567, 568, 569, 819, 595, 567, + 568, 569, 819, 596, 567, 597, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 599, 567, 568, 569, 819, 600, 567, 568, + 569, 819, 601, 567, 568, 569, 819, 602, + 567, 603, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 605, 567, 568, 569, + 819, 606, 567, 568, 569, 819, 607, 567, + 568, 569, 819, 608, 567, 609, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 611, 567, 568, 569, 819, 612, 567, 568, + 569, 819, 613, 567, 568, 569, 819, 614, + 567, 615, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 617, 622, 567, 568, + 569, 819, 618, 567, 568, 569, 819, 619, + 567, 568, 569, 819, 620, 567, 621, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 623, 567, 568, 569, 819, 624, 567, + 568, 569, 819, 625, 567, 626, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 628, 567, 568, 569, 819, 629, 567, 568, + 569, 819, 630, 567, 568, 569, 819, 631, + 567, 632, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 634, 567, 568, 569, + 819, 635, 567, 568, 569, 819, 636, 567, + 568, 569, 819, 637, 567, 638, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 640, 567, 568, 569, 819, 641, 567, 568, + 569, 819, 642, 567, 643, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 645, + 567, 568, 569, 819, 646, 567, 568, 569, + 819, 647, 567, 568, 569, 819, 648, 567, + 568, 569, 819, 649, 567, 650, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 652, 661, 567, 568, 569, 819, 653, 657, + 567, 568, 569, 819, 654, 567, 568, 569, + 819, 655, 567, 656, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 658, 567, + 568, 569, 819, 659, 567, 660, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 662, 567, 568, 569, 819, 663, 567, 568, + 569, 819, 664, 567, 665, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 667, + 567, 568, 569, 819, 668, 567, 568, 569, + 819, 669, 567, 568, 569, 819, 670, 567, + 671, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 673, 567, 568, 569, 819, + 674, 567, 568, 569, 819, 675, 567, 568, + 569, 819, 676, 567, 677, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 679, + 567, 568, 569, 819, 680, 567, 568, 569, + 819, 681, 567, 568, 569, 819, 682, 567, + 683, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 685, 567, 568, 569, 819, + 686, 567, 568, 569, 819, 687, 567, 568, + 569, 819, 688, 567, 689, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 691, + 567, 568, 569, 819, 692, 567, 568, 569, + 819, 693, 567, 568, 569, 819, 694, 567, + 695, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 697, 702, 567, 568, 569, + 819, 698, 567, 568, 569, 819, 699, 567, + 568, 569, 819, 700, 567, 701, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 703, 567, 568, 569, 819, 704, 567, 568, + 569, 819, 705, 567, 706, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 708, + 567, 568, 569, 819, 709, 567, 568, 569, + 819, 710, 567, 568, 569, 819, 711, 567, + 712, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 714, 567, 568, 569, 819, + 715, 567, 568, 569, 819, 716, 567, 568, + 569, 819, 717, 567, 718, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 720, + 567, 568, 569, 819, 721, 567, 568, 569, + 819, 722, 567, 723, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 725, 567, + 568, 569, 819, 726, 567, 568, 569, 819, + 727, 567, 568, 569, 819, 728, 567, 568, + 569, 819, 729, 567, 730, 569, 819, 567, + 568, 569, 819, 567, 732, 733, 819, 731, + 732, 733, 819, 731, 732, 733, 731, 735, + 819, 819, 735, 819, 819, 736, 736, 736, + 819, 819, 819, 819, 819, 739, 819, 819, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, + 746, 747, 746, 759, 760, 746, 761, 762, + 763, 746, 782, 746, 746, 746, 746, 746, + 784, 746, 785, 746, 786, 746, 746, 746, + 1, 2, 746, 3, 3, 3, 3, 746, + 4, 4, 4, 4, 746, 5, 8, 11, + 13, 15, 8, 9, 10, 8, 8, 746, + 19, 746, 20, 20, 20, 20, 19, 746, + 746, 19, 746, 746, 746, 21, 21, 21, + 21, 746, 746, 746, 746, 22, 746, 23, + 24, 26, 746, 746, 27, 746, 28, 28, + 28, 28, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 746, 29, 32, 35, 746, + 764, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 769, 746, 746, 746, 746, + 746, 746, 746, 746, 746, 746, 770, 746, + 746, 746, 771, 746, 776, 746, 746, 777, + 778, 746, 746, 746, 746, 746, 746, 779, + 746, 766, 768, 746, 765, 746, 746, 746, + 767, 768, 746, 768, 768, 746, 768, 746, + 746, 746, 746, 38, 39, 40, 41, 774, + 746, 773, 746, 746, 746, 775, 746, 746, + 746, 50, 52, 54, 746, 56, 746, 746, + 746, 781, 780, 780, 780, 746, 746, 746, + 746, 746, 58, 58, 58, 746, 59, 746, + 746, 746, 746, 746, 746, 746, 746, 62, + 746, 746, 63, 746, 787, 789, 788, 787, 65, 787, 66, 68, 65, 791, 792, 793, 795, 796, 797, 798, 800, 801, 802, 803, 804, 806, 807, 808, 809, 810, 811, 812, @@ -3193,25 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 823, 824, 819, 819, 819, 819, 820, - 819, 821, 819, 822, 819, 819, 819, 819, - 819, 819, 819, 563, 819, 819, 564, 819, - 566, 569, 734, 819, 825, 819, 819, 819, - 819, 819, 829, 819, 819, 819, 819, 819, - 819, 819, 830, 819, 819, 819, 819, 819, - 819, 819, 831, 832, 819, 819, 819, 819, - 819, 819, 833, 827, 819, 819, 819, 819, - 826, 819, 819, 819, 828, 819, 819, 819, - 819, 819, 819, 737, 819, 819, 819, 835, - 834, 834, 834, 819, 819, 819, 819, 819, - 739, 739, 739, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 842, 838, 838, - 838, 839, 838, 840, 838, 841, 838, 838, - 838, 838, 838, 838, 838, 740, 838, 838, - 741, 838, 838, 838, 847, 843, 843, 843, - 844, 843, 845, 843, 846, 843, 843, 843, - 843, 843, 843, 843, 743, 843, 843, 744, - 843, 843, 843, 848, 848, 849, 849, 0, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3219,8 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 843, 769, 843, 843, 0, 843, 843, 771, - 843, 827, 631, 731, 723, 45, 903, 903, - 897, 45, 912, 900, 903, 729, 741, 0, - 45, 45, 923, 923, 923, 923, 737, 45, - 765, 747, 0, 0, 0, 0, 841, 743, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 839, 0, 3, 3, 3, 3, 839, - 761, 839, 761, 3, 3, 3, 3, 839, - 759, 3, 870, 3, 3, 839, 761, 3, - 870, 3, 3, 839, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 751, - 0, 749, 839, 755, 753, 839, 0, 757, - 3, 3, 3, 3, 839, 0, 0, 0, - 0, 839, 757, 0, 0, 0, 0, 839, - 757, 3, 3, 3, 3, 839, 745, 0, - 841, 725, 727, 27, 27, 27, 27, 841, - 743, 3, 3, 3, 3, 841, 763, 0, - 841, 3, 3, 3, 3, 841, 3, 3, - 3, 3, 841, 673, 3, 3, 3, 3, - 841, 0, 0, 0, 0, 841, 27, 27, - 27, 27, 841, 725, 0, 727, 25, 25, - 25, 25, 841, 0, 0, 829, 0, 0, - 0, 603, 0, 0, 0, 0, 0, 0, - 829, 0, 0, 0, 601, 0, 0, 0, - 0, 0, 0, 829, 0, 0, 0, 603, - 0, 0, 0, 0, 675, 0, 17, 831, - 675, 0, 7, 3, 870, 3, 3, 831, - 17, 831, 663, 17, 831, 663, 17, 831, - 663, 831, 3, 661, 870, 3, 3, 831, - 3, 661, 870, 3, 3, 831, 3, 661, - 3, 3, 3, 831, 3, 665, 3, 3, - 3, 831, 3, 3, 3, 3, 837, 671, - 3, 3, 3, 3, 837, 3, 3, 3, - 3, 837, 669, 3, 3, 3, 3, 837, - 3, 3, 3, 3, 837, 3, 667, 3, - 3, 3, 837, 0, 833, 677, 0, 833, - 679, 0, 0, 0, 835, 0, 29, 11, - 845, 31, 13, 845, 0, 13, 845, 53, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, + 3, 3, 3, 841, 3, 3, 3, 3, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 0, 0, 539, 0, 0, 0, - 377, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 435, 0, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 381, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 385, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 389, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 393, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 397, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 401, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 405, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 409, 0, 0, 0, 539, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 413, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 417, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 421, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 425, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 429, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 433, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 379, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 383, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 387, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 391, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 395, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 399, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 403, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 407, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 411, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 415, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 419, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 423, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 427, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 431, 0, 0, 0, 539, - 0, 0, 0, 377, 0, 0, 0, 0, - 0, 533, 471, 0, 533, 475, 0, 0, - 0, 535, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 777, 773, 777, 0, - 777, 45, 777, 45, 777, 773, 775, 777, - 823, 767, 823, 823, 0, 823, 823, 0, - 823, 0, 0, 779, 0, 0, 0, 0, - 821, 3, 3, 3, 3, 821, 1, 3, - 1, 0, 3, 3, 855, 855, 3, 3, - 819, 0, 821, 27, 27, 27, 27, 0, - 821, 745, 0, 821, 735, 733, 3, 3, - 3, 3, 821, 739, 821, 763, 0, 821, - 1, 1, 0, 821, 725, 0, 727, 25, - 25, 25, 25, 821, 615, 613, 783, 619, - 617, 785, 623, 621, 787, 0, 0, 0, - 781, 5, 639, 647, 611, 695, 607, 717, - 699, 715, 683, 683, 0, 605, 713, 691, - 683, 703, 687, 719, 641, 657, 645, 0, - 693, 659, 655, 906, 697, 45, 683, 651, - 45, 0, 653, 689, 649, 683, 701, 685, - 7, 643, 852, 915, 721, 15, 791, 867, - 791, 858, 919, 795, 927, 919, 793, 919, - 847, 709, 711, 681, 0, 7, 0, 849, - 17, 801, 17, 799, 876, 799, 17, 797, - 873, 797, 1, 1, 1, 815, 0, 803, - 705, 707, 45, 19, 23, 21, 805, 882, - 894, 888, 805, 0, 0, 0, 807, 11, - 825, 627, 625, 789, 55, 45, 45, 53, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 435, 0, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, + 0, 0, 377, 0, 0, 0, 0, 0, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,47 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 529, - 501, 529, 529, 0, 529, 529, 0, 529, - 0, 0, 0, 531, 5, 489, 439, 457, - 493, 493, 0, 437, 485, 493, 461, 481, - 451, 441, 0, 487, 453, 449, 495, 455, - 493, 445, 45, 0, 447, 483, 443, 493, - 459, 479, 7, 9, 473, 497, 497, 499, - 15, 517, 861, 517, 15, 519, 864, 519, - 467, 469, 477, 0, 521, 463, 465, 45, - 19, 23, 21, 523, 879, 891, 885, 523, - 0, 0, 0, 525, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 565, 555, 565, 565, 0, 565, 565, - 0, 565, 553, 567, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 583, 573, 583, 583, 0, 583, 583, 0, - 583, 571, 585, 589, 591, 593, 595, 0, - 843, 843, 843, 827, 827, 841, 841, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 841, 841, 841, - 841, 841, 841, 841, 841, 841, 841, 829, - 829, 829, 829, 829, 829, 829, 829, 829, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 831, 831, 831, 837, 837, 837, 837, - 837, 837, 833, 833, 835, 845, 845, 845, - 59, 59, 59, 59, 59, 59, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3691,7 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 537, 537, 537, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 533, 533, 535, 569, 569, 569, 587, 587, - 587, 0, 823, 823, 823, 779, 821, 821, - 819, 821, 821, 821, 821, 821, 821, 821, - 821, 783, 785, 787, 781, 817, 791, 791, - 795, 793, 847, 811, 809, 801, 799, 799, - 797, 797, 815, 803, 813, 805, 805, 807, - 825, 789, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 529, 529, 529, 531, 531, - 517, 517, 519, 519, 513, 527, 521, 515, - 523, 523, 525, 0, 551, 0, 565, 565, - 565, 567, 0, 583, 583, 583, 585, 0, - 0, 0 + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp index e0393e4935561..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp @@ -74,28 +74,28 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { - 41, 65, 66, 67, 76, 78, 85, 41, - 41, 78, 41, 89, 41, 67, 41, 82, - 41, 76, 41, 70, 41, 41, 83, 41, - 82, 41, 95, 41, 65, 85, 41, 78, - 41, 89, 41, 67, 41, 78, 41, 73, - 41, 67, 41, 79, 41, 68, 41, 69, - 41, 82, 41, 76, 41, 70, 73, 41, - 77, 41, 73, 41, 84, 41, 95, 41, - 77, 82, 41, 65, 41, 84, 41, 67, - 41, 72, 41, 61, 41, 48, 57, 41, - 48, 57, 41, 69, 41, 67, 41, 85, - 41, 82, 41, 83, 41, 73, 41, 79, - 41, 78, 41, 79, 41, 95, 41, 65, - 83, 41, 85, 41, 84, 41, 79, 41, - 95, 41, 80, 41, 79, 41, 83, 41, - 83, 41, 69, 41, 83, 41, 83, 41, - 84, 41, 65, 41, 82, 41, 84, 41, - 95, 41, 79, 41, 80, 41, 84, 41, - 67, 84, 41, 80, 41, 41, 70, 41, - 49, 51, 56, 41, 54, 41, 50, 41, - 40, 42, 0 + static const unsigned char _ControlVerbs_trans_keys[] = { + 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, + 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, + 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, + 82u, 41u, 95u, 41u, 65u, 85u, 41u, 78u, + 41u, 89u, 41u, 67u, 41u, 78u, 41u, 73u, + 41u, 67u, 41u, 79u, 41u, 68u, 41u, 69u, + 41u, 82u, 41u, 76u, 41u, 70u, 73u, 41u, + 77u, 41u, 73u, 41u, 84u, 41u, 95u, 41u, + 77u, 82u, 41u, 65u, 41u, 84u, 41u, 67u, + 41u, 72u, 41u, 61u, 41u, 48u, 57u, 41u, + 48u, 57u, 41u, 69u, 41u, 67u, 41u, 85u, + 41u, 82u, 41u, 83u, 41u, 73u, 41u, 79u, + 41u, 78u, 41u, 79u, 41u, 95u, 41u, 65u, + 83u, 41u, 85u, 41u, 84u, 41u, 79u, 41u, + 95u, 41u, 80u, 41u, 79u, 41u, 83u, 41u, + 83u, 41u, 69u, 41u, 83u, 41u, 83u, 41u, + 84u, 41u, 65u, 41u, 82u, 41u, 84u, 41u, + 95u, 41u, 79u, 41u, 80u, 41u, 84u, 41u, + 67u, 84u, 41u, 80u, 41u, 41u, 70u, 41u, + 49u, 51u, 56u, 41u, 54u, 41u, 50u, 41u, + 40u, 42u, 0u }; static const signed char _ControlVerbs_single_lengths[] = { @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { From 514af293be2dc2e628e360431d2972408a89ac62 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 03:37:45 +0000 Subject: [PATCH 15/81] Backport #125505 to 26.8: Fix a possible segmentation fault in `AvroConfluent` when reading an Avro `map` into a `Map` column --- .../Formats/Impl/AvroRowInputFormat.cpp | 5 +- .../Formats/Impl/AvroRowInputFormat.h | 3 +- ..._avro_confluent_map_datetime_key.reference | 3 + .../05385_avro_confluent_map_datetime_key.sh | 67 +++++++++++++++++++ 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.reference create mode 100755 tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.sh diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 366af904b8d5f..7b4c6afd92b31 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -1378,9 +1378,8 @@ const AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(Sc schema_id, format_settings.avro.schema_registry_timeouts, format_settings.avro.schema_registry_retry); - AvroDeserializer deserializer( - output.getHeader(), schema, format_settings.avro.allow_missing_fields, format_settings.null_as_default, format_settings); - it = deserializer_cache.emplace(schema_id, std::move(deserializer)).first; + it = deserializer_cache.try_emplace( + schema_id, output.getHeader(), schema, format_settings.avro.allow_missing_fields, format_settings.null_as_default, format_settings).first; } return it->second; } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index 7930d47004539..23ea0955d3f13 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -56,9 +56,10 @@ class AvroDeserializer AvroDeserializer(const Block & header, avro::ValidSchema schema, bool allow_missing_fields, bool null_as_default_, const FormatSettings & settings_); AvroDeserializer(DataTypePtr data_type, const std::string & column_name, avro::ValidSchema schema, bool allow_missing_fields, bool null_as_default_, const FormatSettings & settings_); + /// Not movable: stored closures capture `this` and references into `symbolic_skip_fn_map`. AvroDeserializer(const AvroDeserializer &) = delete; AvroDeserializer & operator=(const AvroDeserializer &) = delete; - AvroDeserializer(AvroDeserializer &&) = default; + AvroDeserializer(AvroDeserializer &&) = delete; AvroDeserializer & operator=(AvroDeserializer &&) = delete; void deserializeRow(MutableColumns & columns, avro::Decoder & decoder, RowReadExtension & ext) const; diff --git a/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.reference b/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.reference new file mode 100644 index 0000000000000..1ce50d0301a6d --- /dev/null +++ b/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.reference @@ -0,0 +1,3 @@ +1 {'2020-01-02 03:04:05':'a','2021-06-07 08:09:10':'b'} +2 {'1999-12-31 23:59:59':'c'} +3 {} diff --git a/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.sh b/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.sh new file mode 100755 index 0000000000000..23327230bfe89 --- /dev/null +++ b/tests/queries/0_stateless/05385_avro_confluent_map_datetime_key.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the Avro format is not built in the fast test. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# AvroConfluent reads an Avro map into `Map(DateTime, String)` the same way Avro does. + +SCHEMA='{"type":"record","name":"r","fields":[{"name":"id","type":"long"},{"name":"m","type":{"type":"map","values":"string"}}]}' + +PORT_FILE=$(mktemp "${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.XXXXXX.port") + +# A schema registry that serves the schema above for any id. It binds to port 0 and reports the port +# it got, so parallel runs cannot collide. +python3 -c " +import json, os, sys +from http.server import HTTPServer, BaseHTTPRequestHandler +body = json.dumps({'schema': sys.argv[1]}).encode() +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = HTTPServer(('127.0.0.1', 0), Handler) +with open(sys.argv[2] + '.tmp', 'w') as f: + f.write(str(server.server_address[1])) +os.replace(sys.argv[2] + '.tmp', sys.argv[2]) +server.serve_forever() +" "$SCHEMA" "$PORT_FILE" & +HTTP_PID=$! +trap 'kill $HTTP_PID 2>/dev/null; wait $HTTP_PID 2>/dev/null; rm -f "$PORT_FILE"' EXIT + +for _ in $(seq 1 300); do + [[ -s "$PORT_FILE" ]] && break + sleep 0.1 +done +PORT=$(cat "$PORT_FILE") + +# Three messages framed with magic byte 0 and schema id 1. +python3 -c " +import struct, sys +def zz(v): + v = (v << 1) ^ (v >> 63) + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + out.append(b | 0x80 if v else b) + if not v: + return bytes(out) +def s(x): + return zz(len(x)) + x.encode() +out = bytearray() +for rid, m in [(1, {'2020-01-02 03:04:05': 'a', '2021-06-07 08:09:10': 'b'}), (2, {'1999-12-31 23:59:59': 'c'}), (3, {})]: + out += b'\x00' + struct.pack('>I', 1) + zz(rid) + if m: + out += zz(len(m)) + b''.join(s(k) + s(v) for k, v in m.items()) + out += zz(0) +sys.stdout.buffer.write(bytes(out)) +" | $CLICKHOUSE_LOCAL --input-format AvroConfluent --format_avro_schema_registry_url "http://127.0.0.1:$PORT" \ + -S "id Int64, m Map(DateTime('UTC'), String)" -q "SELECT id, m FROM table ORDER BY id" From 64c27aa0c2694eb4cf6866541a25323a04089414 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 09:37:01 +0000 Subject: [PATCH 16/81] Backport #123133 to 26.8: Require permission to override named collections --- .../server-config/named-collections.mdx | 38 +-- src/Backups/BackupInfo.cpp | 14 +- .../NamedCollections/NamedCollections.h | 5 + src/Core/Settings.cpp | 10 +- src/Core/SettingsChangesHistory.cpp | 1 + src/Databases/MySQL/DatabaseMySQL.cpp | 8 +- src/Storages/Kafka/StorageKafkaUtils.cpp | 8 +- src/Storages/NATS/StorageNATS.cpp | 60 +---- src/Storages/NamedCollectionsHelpers.cpp | 228 +++++++++++++----- src/Storages/NamedCollectionsHelpers.h | 31 ++- .../ObjectStorage/S3/Configuration.cpp | 4 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 8 +- src/Storages/StorageDictionary.cpp | 16 ++ src/Storages/StorageMySQL.cpp | 10 +- src/Storages/StorageMySQL.h | 5 +- src/Storages/StoragePostgreSQL.cpp | 11 +- src/Storages/StoragePostgreSQL.h | 5 +- src/Storages/StorageURL.cpp | 6 +- .../TableFunctionPostgreSQL.cpp | 5 +- src/TableFunctions/TableFunctionURL.cpp | 4 + .../configs/users.xml | 7 + .../test_arrowflight_storage/test.py | 1 + .../configs/users.xml | 7 + .../test.py | 1 + .../test_backup_restore_s3/configs/users.xml | 7 + .../test_backup_restore_s3/test.py | 1 + .../configs/user_admin.xml | 2 + .../integration/test_dictionaries_ddl/test.py | 7 +- .../test_dictionaries_mysql/configs/users.xml | 1 + .../configs/users.xml | 3 +- .../test_dictionaries_postgresql/test.py | 1 + .../configs/users.xml | 1 + .../configs/users.xml | 1 + .../configs/users.xml | 1 + .../configs/users.xml | 1 + .../test_postgresql_ssl/configs/users.xml | 1 + .../test_s3_client_refresh/configs/users.xml | 8 + .../test_s3_client_refresh/test.py | 1 + .../configs/users.xml | 1 + .../test_storage_bigquery/configs/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../test_storage_dict/configs/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + .../test_storage_mysql/configs/users.xml | 1 + tests/integration/test_storage_mysql/test.py | 26 -- .../test_storage_postgresql/configs/users.xml | 1 + .../test_storage_url/configs/users.xml | 1 + .../configs/users.xml | 1 + ...02883_named_collections_override.reference | 18 +- .../02883_named_collections_override.sh | 53 ++-- ...4665_nats_credentials_named_collection.sql | 12 - ..._nats_credential_file_path_restriction.sql | 22 -- ...named_collection_override_access.reference | 11 + .../05296_named_collection_override_access.sh | 100 ++++++++ ...llection_override_access_sources.reference | 0 ...amed_collection_override_access_sources.sh | 105 ++++++++ ...collection_generated_args_access.reference | 20 ++ ..._named_collection_generated_args_access.sh | 89 +++++++ ...ction_dictionary_override_access.reference | 16 ++ ...d_collection_dictionary_override_access.sh | 140 +++++++++++ ...ion_kafka_nested_override_access.reference | 6 + ...collection_kafka_nested_override_access.sh | 73 ++++++ ...tgresql_function_settings_access.reference | 8 + ...ion_postgresql_function_settings_access.sh | 69 ++++++ 69 files changed, 1034 insertions(+), 277 deletions(-) create mode 100644 tests/integration/test_arrowflight_storage/configs/users.xml create mode 100644 tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml create mode 100644 tests/integration/test_backup_restore_s3/configs/users.xml create mode 100644 tests/integration/test_s3_client_refresh/configs/users.xml create mode 100644 tests/queries/0_stateless/05296_named_collection_override_access.reference create mode 100755 tests/queries/0_stateless/05296_named_collection_override_access.sh create mode 100644 tests/queries/0_stateless/05297_named_collection_override_access_sources.reference create mode 100755 tests/queries/0_stateless/05297_named_collection_override_access_sources.sh create mode 100644 tests/queries/0_stateless/05315_named_collection_generated_args_access.reference create mode 100755 tests/queries/0_stateless/05315_named_collection_generated_args_access.sh create mode 100644 tests/queries/0_stateless/05316_named_collection_dictionary_override_access.reference create mode 100755 tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh create mode 100644 tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.reference create mode 100755 tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.sh create mode 100644 tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.reference create mode 100755 tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.sh diff --git a/docs/concepts/features/configuration/server-config/named-collections.mdx b/docs/concepts/features/configuration/server-config/named-collections.mdx index 56726e6f6fac5..1bbf1acf11c8c 100644 --- a/docs/concepts/features/configuration/server-config/named-collections.mdx +++ b/docs/concepts/features/configuration/server-config/named-collections.mdx @@ -22,16 +22,15 @@ The keys in a named collection must match the parameter names of the correspondi function, table engine, database, etc. In the examples below the parameter list is linked to for each type. -Parameters set in a named collection can be overridden in SQL, this is shown in the examples -below. This ability can be limited using `[NOT] OVERRIDABLE` keywords and XML attributes -and/or the configuration option `allow_named_collection_override_by_default`. +Overriding a parameter stored in a named collection requires the `SHOW NAMED COLLECTIONS SECRETS` privilege on that collection, including when the parameter is marked `OVERRIDABLE`. Keys marked `NOT OVERRIDABLE` in SQL or `overridable="false"` in XML cannot be overridden. Adding a parameter that the collection does not store does not require this additional privilege. Aliases, such as `addresses_expr` for `host`, count as overrides of the stored parameter. - -If override is allowed, it may be possible for users without administrative access to -figure out the credentials that you are trying to hide. -If you are using named collections with that purpose, you should disable -`allow_named_collection_override_by_default` (which is enabled by default). - +```sql +GRANT SHOW NAMED COLLECTIONS SECRETS ON my_collection TO my_user; +``` + +This check requires the privilege alone; it does not depend on the settings that control displaying secrets. The `allow_named_collection_override_by_default` setting is obsolete and has no effect. + +Dictionary sources follow the same rule. The privilege is checked when the dictionary is created, attached, or restored. When the dictionary is loaded, only keys marked `NOT OVERRIDABLE` are enforced. ## Storing named collections in the system database {#storing-named-collections-in-the-system-database} @@ -46,9 +45,9 @@ url = 'https://connection.url/' In the above example: -* `key_1` can always be overridden. +* `key_1` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. * `key_2` can never be overridden. -* `url` can be overridden or not depending on the value of `allow_named_collection_override_by_default`. +* `url` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. ### Permissions to create named collections with DDL {#permissions-to-create-named-collections-with-ddl} @@ -123,9 +122,9 @@ Changing the storage type requires a server restart; `SYSTEM RELOAD CONFIG` does In the above example: -* `key_1` can always be overridden. +* `key_1` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. * `key_2` can never be overridden. -* `url` can be overridden or not depending on the value of `allow_named_collection_override_by_default`. +* `url` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. ## Modifying named collections {#modifying-named-collections} @@ -488,9 +487,16 @@ SELECT * FROM remote(remote1, database = default, table = test); ### Example of using named collections with a dictionary with source ClickHouse {#example-of-using-named-collections-with-a-dictionary-with-source-clickhouse} ```sql +CREATE NAMED COLLECTION remote_dict AS +host = 'localhost', +port = 9000, +database = 'default', +user = 'default', +password = ''; + CREATE DICTIONARY dict(a Int64, b String) PRIMARY KEY a -SOURCE(CLICKHOUSE(NAME remote1 TABLE test DB default)) +SOURCE(CLICKHOUSE(NAME remote_dict TABLE test)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); @@ -638,7 +644,7 @@ CREATE DICTIONARY dict `b` String ) PRIMARY KEY a -SOURCE(MONGODB(NAME mymongo COLLECTION my_dict)) +SOURCE(MONGODB(NAME mymongo)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()) @@ -650,5 +656,5 @@ SELECT dictGet('dict', 'b', 2); ``` -The named collection specifies `my_collection` for the collection name. In the function call it is overwritten by `collection = 'my_dict'` to select another collection. +The dictionary uses `my_collection`, as specified in the named collection. To use another MongoDB collection, define a separate named collection with that value. diff --git a/src/Backups/BackupInfo.cpp b/src/Backups/BackupInfo.cpp index 5b68b55a10e4f..eb15b02364a01 100644 --- a/src/Backups/BackupInfo.cpp +++ b/src/Backups/BackupInfo.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -22,11 +21,6 @@ namespace DB { -namespace Setting -{ - extern const SettingsBool allow_named_collection_override_by_default; -} - namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -515,15 +509,9 @@ NamedCollectionPtr BackupInfo::getNamedCollection(ContextPtr context) const { auto mutable_collection = collection->duplicate(); auto params_from_query = getParamsMapFromAST(kv_args, context); - const auto allow_override_by_default = context->getSettingsRef()[Setting::allow_named_collection_override_by_default]; for (const auto & [key, value] : params_from_query) { - /// Enforce the same override permission as the table-function/storage paths - /// (`tryGetNamedCollectionWithOverrides`): a non-overridable key (e.g. an operator-static endpoint or - /// credentials) must not be redirected from the query, otherwise the collection's static credentials - /// could be reused against a user-chosen endpoint under the S3 credential restriction. - if (!mutable_collection->isOverridable(key, allow_override_by_default)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed for '{}'", key); + checkNamedCollectionOverride(*collection, key, context); /// Marked before the value is written so the mark remembers the replaced stored value. mutable_collection->markQueryOverridden(key); mutable_collection->setOrUpdate(key, fieldToString(value), {}); diff --git a/src/Common/NamedCollections/NamedCollections.h b/src/Common/NamedCollections/NamedCollections.h index 937464d8bd179..d38129ab01159 100644 --- a/src/Common/NamedCollections/NamedCollections.h +++ b/src/Common/NamedCollections/NamedCollections.h @@ -38,6 +38,11 @@ class NamedCollection bool has(const Key & key) const; + const std::string & getName() const + { + return collection_name; + } + bool hasAny(const std::initializer_list & keys) const; template T get(const Key & key) const; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 6e8a0616fa0b6..1267bae62bc33 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6445,6 +6445,8 @@ For example, if `url_base` is `https://example.com/def/`, then: - `data.csv` resolves to `https://example.com/def/data.csv` - `/test/data.csv` resolves to `https://example.com/test/data.csv` - `//other.com/test/data.csv` resolves to `https://other.com/test/data.csv` + +When the relative URL comes from a [named collection](/concepts/features/configuration/server-config/named-collections), resolving it counts as an override of the `url` key of the collection and requires the `SHOW NAMED COLLECTIONS SECRETS` privilege on that collection. )", 0) \ DECLARE(String, s3_base, "", R"( The base URL used to resolve relative URLs in the [s3](../../sql-reference/table-functions/s3.md) table function and the [S3](../../engines/table-engines/integrations/s3.md) table engine, as well as in the table functions that share their configuration (`s3Cluster`, `gcs`, `oss`). @@ -6454,6 +6456,8 @@ When set, a URL without a scheme is resolved against `s3_base` per RFC 3986, usi For example, if `s3_base` is `s3://clickhouse-public-datasets/`, then `s3('hits_compatible/hits.csv')` reads `s3://clickhouse-public-datasets/hits_compatible/hits.csv`. The base URL can use any form accepted by the `s3` table function, e.g. `s3://bucket/`, `https://bucket.s3.amazonaws.com/` or `https://endpoint/bucket/`. + +When the relative URL comes from a [named collection](/concepts/features/configuration/server-config/named-collections), resolving it counts as an override of the `url` key of the collection and requires the `SHOW NAMED COLLECTIONS SECRETS` privilege on that collection. )", 0) \ DECLARE(UInt64, database_replicated_initial_query_timeout_sec, 300, R"( Sets how long initial DDL query should wait for Replicated database to process previous DDL queue entries in seconds. @@ -8047,9 +8051,6 @@ a Tuple( )", 0) \ DECLARE(Bool, create_table_empty_primary_key_by_default, true, R"( Allow to create *MergeTree tables with empty primary key when ORDER BY and PRIMARY KEY not specified -)", 0) \ - DECLARE(Bool, allow_named_collection_override_by_default, true, R"( -Allow named collections' fields override by default. )", 0) \ DECLARE(SQLSecurityType, default_normal_view_sql_security, SQLSecurityType::INVOKER, R"( Allows to set default `SQL SECURITY` option while creating a normal view. [More about SQL security](/reference/statements/create/view#sql_security). @@ -9364,7 +9365,8 @@ Enable experimental table function `eval`. MAKE_OBSOLETE(M, Bool, use_text_index_dictionary_cache, false) \ MAKE_OBSOLETE(M, Bool, query_plan_use_logical_join_step, true) \ MAKE_OBSOLETE(M, Bool, query_plan_use_new_logical_join_step, true) \ - MAKE_OBSOLETE(M, UInt64, cloud_mode_database_engine, 1) + MAKE_OBSOLETE(M, UInt64, cloud_mode_database_engine, 1) \ + MAKE_OBSOLETE(M, Bool, allow_named_collection_override_by_default, true) /** The section above is for obsolete settings. Do not add anything there. */ #endif /// __CLION_IDE__ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a33162430ed93..f3da652845c0d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -47,6 +47,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8", { + {"allow_named_collection_override_by_default", true, true, "Obsolete. Overriding named collection keys requires `SHOW NAMED COLLECTIONS SECRETS`."}, {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.8 restores the previous strict behavior."}, {"analyzer_compatibility_allow_cte_redefinition", false, false, "New compatibility setting. When enabled, the analyzer accepts a CTE name defined more than once in a single `WITH` clause and lets a later definition shadow the earlier ones, as the query analysis before v24.3 did."}, {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 9ebb8dcaeb812..8c93a25e5800c 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -639,7 +639,13 @@ void registerDatabaseMySQL(DatabaseFactory & factory) ASTs & arguments = engine->arguments->children; auto mysql_settings = std::make_unique(); - if (auto named_collection = tryGetNamedCollectionWithOverrides(arguments, args.context)) + if (auto named_collection = tryGetNamedCollectionWithOverrides( + arguments, + args.context, + /*throw_unknown_collection=*/ true, + /*complex_args=*/ nullptr, + /*dependent_table_id=*/ nullptr, + engine_define->settings)) { configuration = StorageMySQL::processNamedCollectionResult(*named_collection, *mysql_settings, args.context, false); } diff --git a/src/Storages/Kafka/StorageKafkaUtils.cpp b/src/Storages/Kafka/StorageKafkaUtils.cpp index 385f81b037b94..6e5880d9aca74 100644 --- a/src/Storages/Kafka/StorageKafkaUtils.cpp +++ b/src/Storages/Kafka/StorageKafkaUtils.cpp @@ -116,7 +116,13 @@ void registerStorageKafka(StorageFactory & factory) auto kafka_settings = std::make_unique(); String collection_name; - if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext(), true, nullptr, &args.table_id)) + if (auto named_collection = tryGetNamedCollectionWithOverrides( + args.engine_args, + args.getLocalContext(), + /*throw_unknown_collection=*/ true, + /*complex_args=*/ nullptr, + &args.table_id, + args.storage_def->settings)) { kafka_settings->loadFromNamedCollection(named_collection); collection_name = assert_cast(args.engine_args[0].get())->name(); diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index 7246b4b60e689..b4ef9dde78b9c 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -46,7 +46,6 @@ namespace DB { namespace Setting { -extern const SettingsBool allow_named_collection_override_by_default; extern const SettingsNonZeroUInt64 max_insert_block_size; extern const SettingsMilliseconds rabbitmq_max_wait_ms; extern const SettingsMilliseconds stream_flush_interval_ms; @@ -1050,7 +1049,6 @@ bool resolveCredentialSource( bool password_assigned_by_query, bool token_assigned_by_query, bool destination_assigned_by_query, - bool allow_named_collection_override_by_default, bool loading_from_existing_metadata) { /// The value the named collection defines itself, before a query override of the same key. @@ -1120,55 +1118,6 @@ bool resolveCredentialSource( ErrorCodes::BAD_ARGUMENTS, "`nats_url` and `nats_server_list` cannot be overridden when credentials come from the server configuration file"); - /// Credentials the operator explicitly locked (``) cannot be - /// replaced from a query. `tryGetNamedCollectionWithOverrides` checks this for the engine-argument - /// spelling - `nats_credentials` inherits the permission of the path key it replaces, see - /// `findOverrideForbiddingKey` - but the `SETTINGS` clause is applied on top of the collection values - /// without passing through that check, so the permission is enforced here for both spellings. It is - /// enforced when loading from metadata as well, for the same reason it is enforced there for the - /// engine-argument spelling: the lock is a policy on a named collection that is still in use, and the - /// alternative is to authenticate with credentials the operator forbade. This must be based on key - /// existence, not the value: `tryGetNamedCollectionWithOverrides` also refuses a new key when - /// `allow_named_collection_override_by_default` is disabled. When the collection already stores - /// `nats_credentials`, this is a same-key override and uses `allow_named_collection_override_by_default`. - /// Replacing `nats_credential_file` uses `true`: passing the contents is the only way to supply these - /// credentials from SQL, so the operator states the permission with the attribute. - if (named_collection) - { - /// This exactly mirrors `findOverrideForbiddingKey`: inline credentials replace a configured - /// file path, but otherwise they are either a same-key override or a new key. The collection - /// has already been mutated by the engine-argument override, so use its pre-override state. - const auto is_defined_in_collection = [&](const std::string & key) - { - return named_collection->isQueryOverridden(key) ? named_collection->getValueBeforeQueryOverride(key).has_value() - : named_collection->has(key); - }; - const auto check_override_allowed = [&](const char * key, bool default_value) - { - if (!named_collection->isOverridable(key, default_value)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed for '{}'", key); - }; - - if (credentials_assigned_by_query) - { - const auto * key = is_defined_in_collection("nats_credentials") || !is_defined_in_collection("nats_credential_file") - ? "nats_credentials" - : "nats_credential_file"; - check_override_allowed(key, std::string_view{key} == "nats_credential_file" || allow_named_collection_override_by_default); - } - - /// `nats_username`, `nats_password`, and `nats_token` do not have an alternative spelling, - /// so their query overrides follow the regular named-collection policy. In particular, this - /// prevents a `SETTINGS` clause from clearing operator-provided credentials and bypassing the - /// destination-binding check above. - if (username_assigned_by_query) - check_override_allowed("nats_username", allow_named_collection_override_by_default); - if (password_assigned_by_query) - check_override_allowed("nats_password", allow_named_collection_override_by_default); - if (token_assigned_by_query) - check_override_allowed("nats_token", allow_named_collection_override_by_default); - } - /// A path to a credentials file is only accepted from the server configuration file: the server opens /// the file with its own privileges, and during authentication the credentials are sent to `nats_url`, /// which comes from the same query. So taking a path from SQL would let anyone who can define a `NATS` @@ -1240,7 +1189,13 @@ void registerStorageNATS(StorageFactory & factory) bool destination_assigned_by_query = false; /// Whether the named collection is defined in the server configuration file rather than created by SQL. bool collection_defined_in_config = false; - auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext(), true, nullptr, &args.table_id); + auto named_collection = tryGetNamedCollectionWithOverrides( + args.engine_args, + args.getLocalContext(), + /*throw_unknown_collection=*/ true, + /*complex_args=*/ nullptr, + &args.table_id, + args.storage_def->settings); if (named_collection) { nats_settings->loadFromNamedCollection(named_collection); @@ -1313,7 +1268,6 @@ void registerStorageNATS(StorageFactory & factory) password_assigned_by_query, token_assigned_by_query, destination_assigned_by_query, - args.getLocalContext()->getSettingsRef()[Setting::allow_named_collection_override_by_default], (isLoadingFromExistingMetadata(args.mode) || args.query.attach_short_syntax) && (!named_collection || collection_defined_in_config)); diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index a1d725736e929..7428ecb60a591 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -1,11 +1,11 @@ #include #include -#include #include #include #include #include #include +#include #include #include #include @@ -13,13 +13,10 @@ #include +#include + namespace DB { -namespace Setting -{ - extern const SettingsBool allow_named_collection_override_by_default; -} - namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -83,15 +80,78 @@ namespace return std::pair{key, Field(value)}; } - /// A credential can be given either as a path to a file (`ssl_ca`, `nats_credential_file`) or as - /// the contents of that file (`ssl_ca_pem`, `nats_credentials`) - two spellings of one setting. - /// Only the contents form is accepted from a query (see `StorageMySQL::getSSLParams`, - /// `StoragePostgreSQL::getSSLParams` and `resolveCredentialSource` in `StorageNATS.cpp`), where it - /// replaces the path inherited from the collection. - /// Returns the path key a contents key replaces, if the key is a contents key. - std::optional credentialsPathKeyFor(const std::string & key) + /// `XMLConfiguration` accepts indexes, escapes and redundant dots in paths. Compare the + /// underlying key names so these spellings cannot disguise an override as a new key. + String normalizeKey(std::string_view key) { - static constexpr std::pair credentials_keys[] = { + String normalized; + for (size_t i = 0; i < key.size(); ++i) + { + if (key[i] == '[') + { + auto end = key.find(']', i); + if (end == std::string_view::npos) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid named collection key '{}'", key); + i = end; + } + else if (key[i] == '\\' && i + 1 < key.size()) + normalized += key[++i]; + else if (key[i] != '.' || (!normalized.empty() && normalized.back() != '.')) + normalized += key[i]; + } + if (!normalized.empty() && normalized.back() == '.') + normalized.pop_back(); + return normalized; + } + + /// The librdkafka property that a `Kafka` key sets, and whether the key is a flat `kafka_*` table setting. + /// `KafkaConfigLoader` sets the properties from nested collection keys (`kafka.`, `kafka.consumer.` + /// and `kafka.producer.`, with `_` converted to `.`), and the non-empty flat settings overwrite them afterwards. + struct KafkaProperty + { + String name; + bool is_flat_setting; + }; + + std::optional getKafkaProperty(std::string_view key) + { + static constexpr auto flat_settings = std::to_array>({ + {"kafka_security_protocol", "security.protocol"}, + {"kafka_sasl_mechanism", "sasl.mechanism"}, + {"kafka_sasl_username", "sasl.username"}, + {"kafka_sasl_password", "sasl.password"}, + {"kafka_compression_codec", "compression.codec"}, + {"kafka_compression_level", "compression.level"}, + {"kafka_autodetect_client_rack", "client.rack"}, + }); + + for (const auto & [setting, property] : flat_settings) + { + if (key == setting) + return KafkaProperty{String(property), true}; + } + + for (const std::string_view prefix : {"kafka.consumer.", "kafka.producer.", "kafka."}) + { + if (!key.starts_with(prefix)) + continue; + + KafkaProperty property{String(key.substr(prefix.size())), false}; + /// `log_level` is the only librdkafka property with an underscore. + if (property.name != "log_level") + std::ranges::replace(property.name, '_', '.'); + return property; + } + return std::nullopt; + } + + bool areEquivalentKeys(std::string_view key, std::string_view other) + { + if (NamedCollectionValidateKey{key} == NamedCollectionValidateKey{other} + || NamedCollectionValidateKey{key} == NamedCollectionValidateKey{other}) + return true; + + static constexpr auto equal_keys = std::to_array>({ {"ssl_ca_pem", "ssl_ca"}, {"ssl_cert_pem", "ssl_cert"}, {"ssl_key_pem", "ssl_key"}, @@ -99,45 +159,98 @@ namespace {"sslcert_pem", "sslcert"}, {"sslkey_pem", "sslkey"}, {"nats_credentials", "nats_credential_file"}, - }; - - for (const auto & [contents_key, path_key] : credentials_keys) + {"nats_url", "nats_server_list"}, + {"rabbitmq_host_port", "rabbitmq_address"}, + {"http_method", "method"}, + {"compression_method", "compression"}, + {"storage_account_url", "connection_string"}, + {"user", "credentials.user"}, + {"password", "credentials.password"}, + }); + + for (const auto & [first, second] : equal_keys) { - if (key == contents_key) - return std::string(path_key); + if ((key == first && other == second) || (key == second && other == first)) + return true; } - return std::nullopt; + /// A flat `Kafka` setting and a nested collection key that set the same librdkafka property replace each other. + const auto kafka_property = getKafkaProperty(key); + const auto other_kafka_property = getKafkaProperty(other); + if (kafka_property && other_kafka_property && kafka_property->name == other_kafka_property->name + && (kafka_property->is_flat_setting || other_kafka_property->is_flat_setting)) + return true; + + /// Configuration values include the text of their descendants. Replacing a subtree, + /// or adding a child of an alias, can therefore replace a stored value as well. + const auto key_root = key.substr(0, key.find('.')); + const auto other_root = other.substr(0, other.find('.')); + if (key_root != key || other_root != other) + return areEquivalentKeys(key_root, other_root); + return false; } - /// Whether an override of `key` at the point of use is permitted by the collection. - /// Returns the key that forbids it - which is not always `key` itself - or nullopt when allowed. - /// - /// A contents form (`ssl_ca_pem`) is not a brand-new key when the collection defines the - /// corresponding path (`ssl_ca`): it replaces it, so the permission is taken from the key it - /// replaces. Passing the contents is the only way to supply such a credential from SQL at all - a - /// path is refused there unconditionally - so the permission is the one the operator states - /// explicitly with `` rather than the value of - /// `allow_named_collection_override_by_default`. `StorageMySQL::getSSLParams` re-checks the very - /// same condition when the replacement happens; without this the two disagree, and the documented - /// SQL-safe form is rejected on exactly the installations that need it. - std::optional findOverrideForbiddingKey(const NamedCollection & collection, const std::string & key, bool default_value) + /// Throws if `key` replaces a stored key that is `NOT OVERRIDABLE`. + /// Returns whether `key` replaces a stored key that requires the privilege `SHOW NAMED COLLECTIONS SECRETS`. + bool checkOverrideLockAndFindStoredKey(const NamedCollection & collection, const std::string & key) { - if (!collection.has(key)) + bool overrides_stored_key = false; + const auto normalized_key = normalizeKey(key); + for (const auto & stored_key : collection.getKeys()) { - auto path_key = credentialsPathKeyFor(key); - if (path_key && collection.has(*path_key)) - { - /// The locked key is the path, so name it rather than the contents form the query used. - if (collection.isOverridable(*path_key, /* default_value= */ true)) - return std::nullopt; - return path_key; - } + if (!areEquivalentKeys(normalized_key, normalizeKey(stored_key))) + continue; + + if (!collection.isOverridable(stored_key, /* default_value= */ true)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed for '{}'", stored_key); + + /// ClickHouse appends the inferred `format` and `structure` to the arguments and parses them again. + /// Replacing the stored value `'auto'` neither hides a stored value nor redirects credentials, so it is not an override. + if ((stored_key == "format" || stored_key == "structure") && collection.getOrDefault(stored_key, "") == "auto") + continue; + + overrides_stored_key = true; } + return overrides_stored_key; + } +} - if (collection.isOverridable(key, default_value)) - return std::nullopt; - return key; +void checkNamedCollectionOverrideLock(const NamedCollection & collection, const std::string & key) +{ + checkOverrideLockAndFindStoredKey(collection, key); +} + +void checkNamedCollectionOverride(const NamedCollection & collection, const std::string & key, ContextPtr context) +{ + if (!context) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Checking an override of named collection key '{}' requires a context", key); + if (checkOverrideLockAndFindStoredKey(collection, key)) + context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS, collection.getName()); +} + +void checkNamedCollectionOverridesInDictionarySource( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) +{ + auto collection_name = config.getString(config_prefix + ".name", ""); + if (collection_name.empty()) + return; + + NamedCollectionFactory::instance().loadIfNot(); + + /// A missing collection is reported when the dictionary source is created. + auto collection = NamedCollectionFactory::instance().tryGet(collection_name); + if (!collection) + return; + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_prefix, keys); + for (const auto & key : keys) + { + /// The 'name' key identifies the named collection itself and is not a data key to override. + if (key == "name") + continue; + + checkNamedCollectionOverride(*collection, key, context); } } @@ -170,7 +283,8 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( ContextPtr context, bool throw_unknown_collection, VectorWithMemoryTracking> * complex_args, - const StorageID * dependent_table_id) + const StorageID * dependent_table_id, + const ASTSetQuery * settings) { if (asts.empty()) return nullptr; @@ -192,6 +306,12 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( if (!collection) return nullptr; + if (settings) + { + for (const auto & change : settings->changes) + checkNamedCollectionOverride(*collection, change.name, context); + } + auto collection_copy = collection->duplicate(); if (asts.size() == 1) @@ -201,23 +321,19 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( return collection_copy; } - const auto allow_override_by_default = context->getSettingsRef()[Setting::allow_named_collection_override_by_default]; - for (auto it = std::next(asts.begin()); it != asts.end(); ++it) { auto value_override = getKeyValueFromASTImpl(*it, /* fallback_to_ast_value */ complex_args != nullptr, context); if (!value_override) { - if (!(*it)->as()) + const auto * function = (*it)->as(); + if (!function) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); - if (allow_override_by_default) - continue; - // if allow_override_by_default is false we don't allow extra arguments - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed because setting allow_override_by_default is disabled"); + checkNamedCollectionOverride(*collection, function->name, context); + continue; } - if (auto forbidding_key = findOverrideForbiddingKey(*collection_copy, value_override->first, allow_override_by_default)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed for '{}'", *forbidding_key); + checkNamedCollectionOverride(*collection, value_override->first, context); if (const ASTPtr * value = std::get_if(&value_override->second)) { @@ -253,15 +369,13 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); - const auto allow_override_by_default = context->getSettingsRef()[Setting::allow_named_collection_override_by_default]; for (const auto & key : keys) { /// The 'name' key identifies the named collection itself and is not a data key to override. if (key == "name") continue; - if (auto forbidding_key = findOverrideForbiddingKey(*collection_copy, key, allow_override_by_default)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Override not allowed for '{}'", *forbidding_key); + checkNamedCollectionOverrideLock(*collection, key); /// The keys of a dictionary created with a DDL query come from the query, so mark them the /// same way as the AST-based overload above: `StorageMySQL::getSSLParams` distinguishes a diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 4bfbf1dabfde3..e7803ac56e424 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -17,23 +17,48 @@ namespace DB { +class ASTSetQuery; + namespace ErrorCodes { extern const int BAD_ARGUMENTS; } +/// Throws `BAD_ARGUMENTS` if `key` replaces a stored key (including an alias) that is `NOT OVERRIDABLE`. +/// Does not check privileges. Used where the override was already authorized when the object was created, +/// for example when a dictionary source is loaded in the background. +void checkNamedCollectionOverrideLock(const NamedCollection & collection, const std::string & key); + +/// Checks the lock as `checkNamedCollectionOverrideLock` does. +/// Then, replacing a stored key (including an alias) requires `SHOW NAMED COLLECTIONS SECRETS` in `context`. +/// Replacing a stored `'auto'` value of `format` or `structure` is exempt, because ClickHouse appends the inferred values itself. +/// The context is required and must not be null: a caller that loads an already authorized object uses `checkNamedCollectionOverrideLock`. +void checkNamedCollectionOverride(const NamedCollection & collection, const std::string & key, ContextPtr context); + +/// Checks the overrides of the stored keys in the source of a dictionary at its creation, attachment or restore. +/// `config_prefix` is the source configuration ".source." (e.g. "dictionary.source.clickhouse"). +/// Every key that replaces a stored key is checked with `checkNamedCollectionOverride` in `context`. +/// Does nothing if the source does not refer to an existing named collection. Does not register dependencies. +void checkNamedCollectionOverridesInDictionarySource( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context); + /// Helper function to get named collection for table engine. /// Table engines have collection name as first argument of ast and other arguments are key-value overrides. -/// If dependent_table_id is provided, registers the table as a dependency of the named collection. +/// If `dependent_table_id` is provided, registers the table as a dependency of the named collection. +/// Checks overrides in `SETTINGS` before the engine applies them. MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( ASTs asts, ContextPtr context, bool throw_unknown_collection = true, VectorWithMemoryTracking> * complex_args = nullptr, - const StorageID * dependent_table_id = nullptr); + const StorageID * dependent_table_id = nullptr, + const ASTSetQuery * settings = nullptr); /// Helper function to get named collection for dictionary source. -/// Dictionaries have collection name as name argument of dict configuration and other arguments are overrides. +/// Dictionaries have the collection name as the `name` argument of their configuration. +/// Other arguments may add missing keys or replace stored keys that are not `NOT OVERRIDABLE`. +/// Privileges are not checked here: they are checked when the dictionary is created +/// (see `checkNamedCollectionOverridesInDictionarySource`). /// Also registers the dictionary as a dependency of the named collection, so that /// DROP NAMED COLLECTION is blocked while the dictionary exists. /// The dictionary's identity is derived from config_prefix, which has the form diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 806825224e3cb..e6a406f961185 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -213,7 +213,11 @@ void S3StorageParsedArguments::fromNamedCollection(const NamedCollection & colle const String raw_collection_url = collection.get("url"); const String collection_url = StorageURL::resolveURLBase(raw_collection_url, settings[Setting::s3_base].value, "s3_base"); if (collection_url != raw_collection_url) + { + /// Resolving against `s3_base` replaces the stored `url`, which could send the stored credentials to another host. + checkNamedCollectionOverride(collection, "url", context); url_overridden_by_base_setting = collection_url; + } auto filename = collection.getOrDefault("filename", ""); if (!filename.empty()) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 7aead829181d5..661bffafb20a2 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -1528,7 +1528,13 @@ void registerStorageRabbitMQ(StorageFactory & factory) { auto rabbitmq_settings = std::make_unique(); - if (auto named_collection = tryGetNamedCollectionWithOverrides(args.engine_args, args.getLocalContext(), true, nullptr, &args.table_id)) + if (auto named_collection = tryGetNamedCollectionWithOverrides( + args.engine_args, + args.getLocalContext(), + /*throw_unknown_collection=*/ true, + /*complex_args=*/ nullptr, + &args.table_id, + args.storage_def->settings)) rabbitmq_settings->loadFromNamedCollection(named_collection); else if (!args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "RabbitMQ engine must have settings"); diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index e326c4084e513..81c711b6f6494 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -23,6 +23,7 @@ #endif #include #include +#include #include @@ -46,6 +47,7 @@ namespace ErrorCodes extern const int DICTIONARY_ALREADY_EXISTS; extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } namespace @@ -374,6 +376,20 @@ void registerStorageDictionary(StorageFactory & factory) /// Create dictionary storage that owns underlying dictionary auto abstract_dictionary_configuration = getDictionaryConfigurationFromAST(args.query, local_context, dictionary_id.database_name); + + /// The overrides of stored keys of a named collection need the same privilege as in table engines. + /// The dictionary is loaded later with the global context, so the user's privileges are checked here. + Poco::Util::AbstractConfiguration::Keys source_keys; + abstract_dictionary_configuration->keys("dictionary.source", source_keys); + if (source_keys.empty() || source_keys.size() > 2) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected one or two child elements in the source of dictionary {}", + dictionary_id.getNameForLogs()); + /// The source type is chosen as in `DictionarySourceFactory::create`. + const auto & source_type = source_keys.front() == "settings" ? source_keys.back() : source_keys.front(); + checkNamedCollectionOverridesInDictionarySource( + *abstract_dictionary_configuration, "dictionary.source." + source_type, local_context); auto result_storage = std::make_shared(dictionary_id, abstract_dictionary_configuration, local_context); bool lazy_load = external_dictionaries_loader.isObjectLazy(*abstract_dictionary_configuration, "dictionary") diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 1bb0e8d5bd9f4..dfe38fcda5f94 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -531,10 +531,13 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( return configuration; } -StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings, const StorageID * table_id) +StorageMySQL::Configuration StorageMySQL::getConfiguration( + ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings, + const StorageID * table_id, const ASTSetQuery * settings) { StorageMySQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context_, true, nullptr, table_id)) + if (auto named_collection = tryGetNamedCollectionWithOverrides( + engine_args, context_, /*throw_unknown_collection=*/ true, /*complex_args=*/ nullptr, table_id, settings)) { configuration = StorageMySQL::processNamedCollectionResult(*named_collection, storage_settings, context_); } @@ -610,7 +613,8 @@ void registerStorageMySQL(StorageFactory & factory) factory.registerStorage("MySQL", [](const StorageFactory::Arguments & args) { MySQLSettings mysql_settings; /// TODO: move some arguments from the arguments to the SETTINGS. - auto configuration = StorageMySQL::getConfiguration(args.engine_args, args.getLocalContext(), mysql_settings, &args.table_id); + auto configuration = StorageMySQL::getConfiguration( + args.engine_args, args.getLocalContext(), mysql_settings, &args.table_id, args.storage_def->settings); /// Bridge the query-context value of `mysql_datatypes_support_level` into the engine settings /// (and freeze it into the table definition) so that it is honored during schema inference, diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index 4b5680b38e990..07b2e75709c1d 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -22,6 +22,7 @@ namespace DB struct MySQLSettings; class NamedCollection; +class ASTSetQuery; struct StorageID; /** Implements storage in the MySQL database. @@ -83,7 +84,9 @@ class StorageMySQL final : public StorageWithCommonVirtualColumns, WithContext String addresses_expr; }; - static Configuration getConfiguration(ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings, const StorageID * table_id = nullptr); + static Configuration getConfiguration( + ASTs engine_args, ContextPtr context_, MySQLSettings & storage_settings, + const StorageID * table_id = nullptr, const ASTSetQuery * settings = nullptr); static Configuration processNamedCollectionResult( const NamedCollection & named_collection, MySQLSettings & storage_settings, diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index b8bea7e7b0756..fbe22f7636bf8 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -806,10 +807,13 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult return configuration; } -StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine_args, ContextPtr context, PostgreSQLSettings * storage_settings, const StorageID * table_id) +StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration( + ASTs engine_args, ContextPtr context, PostgreSQLSettings * storage_settings, + const StorageID * table_id, const ASTSetQuery * settings) { StoragePostgreSQL::Configuration configuration; - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context, true, nullptr, table_id)) + if (auto named_collection = tryGetNamedCollectionWithOverrides( + engine_args, context, /*throw_unknown_collection=*/ true, /*complex_args=*/ nullptr, table_id, settings)) { configuration = StoragePostgreSQL::processNamedCollectionResult(*named_collection, storage_settings, context, /*require_table=*/ true); } @@ -881,7 +885,8 @@ void registerStoragePostgreSQL(StorageFactory & factory) PostgreSQLSettings postgresql_settings; postgresql_settings.loadFromQueryContext(*args.getLocalContext()); - auto configuration = StoragePostgreSQL::getConfiguration(args.engine_args, args.getLocalContext(), &postgresql_settings, &args.table_id); + auto configuration = StoragePostgreSQL::getConfiguration( + args.engine_args, args.getLocalContext(), &postgresql_settings, &args.table_id, args.storage_def->settings); if (args.storage_def) postgresql_settings.loadFromQuery(*args.storage_def); diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 23ca9435f8d96..e96ebda74a415 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -23,6 +23,7 @@ using PoolWithFailoverPtr = std::shared_ptr; namespace DB { class NamedCollection; +class ASTSetQuery; struct StorageID; struct PostgreSQLSettings; @@ -79,7 +80,9 @@ class StoragePostgreSQL final : public StorageWithCommonVirtualColumns /// `storage_settings` may be nullptr for callers that do not honor the `PostgreSQLSettings` /// (e.g. the `MaterializedPostgreSQL` engines): the setting names are then rejected in named /// collections instead of being accepted and silently ignored. - static Configuration getConfiguration(ASTs engine_args, ContextPtr context, PostgreSQLSettings * storage_settings, const StorageID * table_id = nullptr); + static Configuration getConfiguration( + ASTs engine_args, ContextPtr context, PostgreSQLSettings * storage_settings, + const StorageID * table_id = nullptr, const ASTSetQuery * settings = nullptr); static Configuration processNamedCollectionResult(const NamedCollection & named_collection, PostgreSQLSettings * storage_settings, ContextPtr context_, bool require_table = true); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index cd59a8c92aa71..a40a081f1d5f9 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -2192,11 +2192,16 @@ void StorageURL::overrideURLInEngineArgs(ASTs & args, const String & resolved_ur StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const ContextPtr & local_context, const StorageID * table_id) { StorageURL::Configuration configuration; + const auto & url_base = local_context->getSettingsRef()[Setting::url_base].value; if (auto named_collection = tryGetNamedCollectionWithOverrides(args, local_context, true, nullptr, table_id)) { StorageURL::processNamedCollectionResult(configuration, *named_collection); evalArgsAndCollectHeaders(args, configuration.headers, local_context, false); + + /// Resolving the stored `url` against `url_base` replaces it, which could send the stored credentials to another host. + if (resolveURLBase(configuration.url, url_base) != configuration.url) + checkNamedCollectionOverride(*named_collection, "url", local_context); } else { @@ -2216,7 +2221,6 @@ StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const Contex /// For the URL engine, the resolved URL is later materialized into the engine args /// AST by `addInferredEngineArgsToCreateQuery`, so DETACH/ATTACH and server restart /// reproduce the originally-resolved URL even if `url_base` is later changed or unset. - const auto & url_base = local_context->getSettingsRef()[Setting::url_base].value; configuration.url = resolveURLBase(configuration.url, url_base); if (configuration.format == "auto") diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 86ebf2211b9d7..679da511f128b 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -117,10 +117,11 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, Contex } } - configuration.emplace(StoragePostgreSQL::getConfiguration(args, context, &postgresql_settings)); + configuration.emplace(StoragePostgreSQL::getConfiguration( + args, context, &postgresql_settings, /*table_id=*/ nullptr, settings_ast ? settings_ast->as() : nullptr)); /// Applied after getConfiguration, so that the explicit SETTINGS clause wins over the values - /// stored in a named collection. + /// stored in a named collection. `getConfiguration` checks these overrides of the collection. if (settings_ast) postgresql_settings.loadFromQuery(settings_ast->as()); diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 1e95ac17dee1c..85deb82048a6b 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -135,6 +135,10 @@ void TableFunctionURL::parseArgumentsImpl(ASTs & args, const ContextPtr & contex format = configuration.format; StorageURL::evalArgsAndCollectHeaders(args, configuration.headers, context); + + /// Resolving the stored `url` against `url_base` replaces it, which could send the stored credentials to another host. + if (StorageURL::resolveURLBase(filename, context->getSettingsRef()[Setting::url_base].value) != filename) + checkNamedCollectionOverride(*named_collection, "url", context); } else { diff --git a/tests/integration/test_arrowflight_storage/configs/users.xml b/tests/integration/test_arrowflight_storage/configs/users.xml new file mode 100644 index 0000000000000..d50d43518fa1b --- /dev/null +++ b/tests/integration/test_arrowflight_storage/configs/users.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_arrowflight_storage/test.py b/tests/integration/test_arrowflight_storage/test.py index 4155816634037..38ca432b6f220 100644 --- a/tests/integration/test_arrowflight_storage/test.py +++ b/tests/integration/test_arrowflight_storage/test.py @@ -9,6 +9,7 @@ node = cluster.add_instance( "node", main_configs=["configs/remote_host_filter.xml"], + user_configs=["configs/users.xml"], with_arrowflight=True, stay_alive=True, ) diff --git a/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml b/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml new file mode 100644 index 0000000000000..d50d43518fa1b --- /dev/null +++ b/tests/integration/test_backup_restore_azure_blob_storage/configs/users.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index b15c5a452ebfe..36e4afda81fbe 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -166,6 +166,7 @@ def cluster(): cluster.add_instance( "node", main_configs=[path], + user_configs=["configs/users.xml"], with_azurite=True, with_zookeeper=True, ) diff --git a/tests/integration/test_backup_restore_s3/configs/users.xml b/tests/integration/test_backup_restore_s3/configs/users.xml new file mode 100644 index 0000000000000..d50d43518fa1b --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/users.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index f756c84a69d5b..0a06846903071 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -152,6 +152,7 @@ def setup_cluster(request): + request.param, user_configs=[ "configs/zookeeper_retries.xml", + "configs/users.xml", ], with_minio=True, # The test compares some S3 events. We disable the remote DB disk, so it doesn't affect the comparing events. diff --git a/tests/integration/test_dictionaries_ddl/configs/user_admin.xml b/tests/integration/test_dictionaries_ddl/configs/user_admin.xml index b5e94aed63a54..6d28d28ac68e4 100644 --- a/tests/integration/test_dictionaries_ddl/configs/user_admin.xml +++ b/tests/integration/test_dictionaries_ddl/configs/user_admin.xml @@ -7,6 +7,8 @@ default default + 1 + 1 diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 6d455b8c07f3f..dd8031348d749 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -623,8 +623,7 @@ def test_secure(started_cluster): # Secure is set to 0 in named collection and in 1 in DDL node1.query("DROP DICTIONARY IF EXISTS test.clickhouse_secure") - node1.query( - """ + create_with_override = """ CREATE DICTIONARY test.clickhouse_secure( id UInt64, value String @@ -638,7 +637,9 @@ def test_secure(started_cluster): )) LIFETIME(MIN 1 MAX 10) """ - ) + # Overriding a stored key requires `SHOW NAMED COLLECTIONS SECRETS`, which `default` does not have + assert "ACCESS_DENIED" in node1.query_and_get_error(create_with_override) + node1.query(create_with_override, user="admin") value = node1.query( "SELECT dictGet('test.clickhouse_secure', 'value', toUInt64(1))" ) diff --git a/tests/integration/test_dictionaries_mysql/configs/users.xml b/tests/integration/test_dictionaries_mysql/configs/users.xml index 70c7d3bc2c1b0..725dfacd7de77 100644 --- a/tests/integration/test_dictionaries_mysql/configs/users.xml +++ b/tests/integration/test_dictionaries_mysql/configs/users.xml @@ -13,6 +13,7 @@ default default 1 + 1 diff --git a/tests/integration/test_dictionaries_postgresql/configs/users.xml b/tests/integration/test_dictionaries_postgresql/configs/users.xml index beb08eb6ed4d2..1c5f4d5a21fc8 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/users.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/users.xml @@ -4,7 +4,8 @@ default default - 1 + 1 + 1 diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 44b669006efb8..a5c0a78222d1d 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -20,6 +20,7 @@ "configs/named_collections.xml", "configs/bg_reconnect.xml", ], + user_configs=["configs/users.xml"], with_postgres=True, with_postgres_cluster=True, ) diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml index d0108b7b97525..5ff9452cb9e3d 100644 --- a/tests/integration/test_mask_sensitive_info/configs/users.xml +++ b/tests/integration/test_mask_sensitive_info/configs/users.xml @@ -14,6 +14,7 @@ default 1 + 1 diff --git a/tests/integration/test_mysql57_database_engine/configs/users.xml b/tests/integration/test_mysql57_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_mysql57_database_engine/configs/users.xml +++ b/tests/integration/test_mysql57_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_mysql_database_engine/configs/users.xml b/tests/integration/test_mysql_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_mysql_database_engine/configs/users.xml +++ b/tests/integration/test_mysql_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_postgresql_database_engine/configs/users.xml b/tests/integration/test_postgresql_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_postgresql_database_engine/configs/users.xml +++ b/tests/integration/test_postgresql_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_postgresql_ssl/configs/users.xml b/tests/integration/test_postgresql_ssl/configs/users.xml index d008a54d3bae7..b83739e0a6306 100644 --- a/tests/integration/test_postgresql_ssl/configs/users.xml +++ b/tests/integration/test_postgresql_ssl/configs/users.xml @@ -4,6 +4,7 @@ 1 + 1 diff --git a/tests/integration/test_s3_client_refresh/configs/users.xml b/tests/integration/test_s3_client_refresh/configs/users.xml new file mode 100644 index 0000000000000..15d5e614fc07c --- /dev/null +++ b/tests/integration/test_s3_client_refresh/configs/users.xml @@ -0,0 +1,8 @@ + + + + 1 + 1 + + + diff --git a/tests/integration/test_s3_client_refresh/test.py b/tests/integration/test_s3_client_refresh/test.py index b9b6a4dabe53c..6223b587ac9bd 100644 --- a/tests/integration/test_s3_client_refresh/test.py +++ b/tests/integration/test_s3_client_refresh/test.py @@ -8,6 +8,7 @@ "node", with_minio=True, main_configs=["configs/s3.xml"], + user_configs=["configs/users.xml"], env_variables={ "AWS_ACCESS_KEY_ID": minio_access_key, "AWS_SECRET_ACCESS_KEY": minio_secret_key, diff --git a/tests/integration/test_s3_credentials_hardening/configs/users.xml b/tests/integration/test_s3_credentials_hardening/configs/users.xml index c313cbd1f94b2..8164b9223f11d 100644 --- a/tests/integration/test_s3_credentials_hardening/configs/users.xml +++ b/tests/integration/test_s3_credentials_hardening/configs/users.xml @@ -2,6 +2,7 @@ 1 + 1 diff --git a/tests/integration/test_storage_bigquery/configs/users.xml b/tests/integration/test_storage_bigquery/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_bigquery/configs/users.xml +++ b/tests/integration/test_storage_bigquery/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_delta/configs/users.d/users.xml b/tests/integration/test_storage_delta/configs/users.d/users.xml index 1cf4851873c51..8ca5fa7963f01 100644 --- a/tests/integration/test_storage_delta/configs/users.d/users.xml +++ b/tests/integration/test_storage_delta/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 1 diff --git a/tests/integration/test_storage_delta_shuffles/configs/users.d/users.xml b/tests/integration/test_storage_delta_shuffles/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_delta_shuffles/configs/users.d/users.xml +++ b/tests/integration/test_storage_delta_shuffles/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_dict/configs/users.xml b/tests/integration/test_storage_dict/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_dict/configs/users.xml +++ b/tests/integration/test_storage_dict/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml index 87f1e61df0891..d9017e13701b2 100644 --- a/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml @@ -9,6 +9,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_schema_evolution/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_schema_evolution/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_schema_evolution/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_schema_evolution/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_mysql/configs/users.xml b/tests/integration/test_storage_mysql/configs/users.xml index a11985dd113d2..845ae0e64ed70 100644 --- a/tests/integration/test_storage_mysql/configs/users.xml +++ b/tests/integration/test_storage_mysql/configs/users.xml @@ -13,6 +13,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index daa8d0d4b8e78..4599dc4459fb9 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -1599,32 +1599,6 @@ def test_mysql_ssl_contents_override_configured_paths(started_cluster): node1.query(f"SELECT count() FROM mysql(mysql_with_locked_ssl, {credentials})") assert "Override not allowed for 'ssl_ca'" in str(exception.value) - # The contents form is the only way to supply a TLS credential from SQL, so it stays usable - # when overrides are forbidden by default: it is not a new key, it replaces the path the - # collection defines, and the operator forbids that with `overridable="false"` instead. - hardened = {"allow_named_collection_override_by_default": 0} - assert ( - node1.query( - f"SELECT count() FROM mysql(mysql_with_ssl, {credentials})", - settings=hardened, - ) - == "0\n" - ) - with pytest.raises(QueryRuntimeException) as exception: - node1.query( - f"SELECT count() FROM mysql(mysql_with_locked_ssl, {credentials})", - settings=hardened, - ) - assert "Override not allowed for 'ssl_ca'" in str(exception.value) - - # An unrelated key is still a new key, and is still refused under that policy. - with pytest.raises(QueryRuntimeException) as exception: - node1.query( - "SELECT count() FROM mysql(mysql_with_ssl, table = 'test_table')", - settings=hardened, - ) - assert "Override not allowed for 'table'" in str(exception.value) - # The same override on the dictionary DDL path: the overrides of `SOURCE(MYSQL(NAME ...))` # arrive as generated configuration keys rather than as an AST, and must be recognized as # query-supplied all the same. The source is instantiated when the dictionary is loaded. diff --git a/tests/integration/test_storage_postgresql/configs/users.xml b/tests/integration/test_storage_postgresql/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_postgresql/configs/users.xml +++ b/tests/integration/test_storage_postgresql/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_url/configs/users.xml b/tests/integration/test_storage_url/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_url/configs/users.xml +++ b/tests/integration/test_storage_url/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_table_function_mongodb/configs/users.xml b/tests/integration/test_table_function_mongodb/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_table_function_mongodb/configs/users.xml +++ b/tests/integration/test_table_function_mongodb/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/queries/0_stateless/02883_named_collections_override.reference b/tests/queries/0_stateless/02883_named_collections_override.reference index 78b3eeda98739..84bbb8c1c4007 100644 --- a/tests/queries/0_stateless/02883_named_collections_override.reference +++ b/tests/queries/0_stateless/02883_named_collections_override.reference @@ -1,32 +1,22 @@ -allow_named_collection_override_by_default=1 u1 +u1 1\n 1\n 1 -allow_named_collection_override_by_default=1 u2 +u2 1\n 1\n 2\n 1 -allow_named_collection_override_by_default=0 u1 -1\n -1 -allow_named_collection_override_by_default=0 u2 -1\n Test ALTER 2\n 1\n Test XML collections -allow_named_collection_override_by_default=1 url_override1 +url_override1 1\n 1\n 1 -allow_named_collection_override_by_default=1 url_override2 +url_override2 1\n 1\n 2\n 1 -allow_named_collection_override_by_default=0 url_override1 -1\n -1 -allow_named_collection_override_by_default=0 url_override2 -1\n diff --git a/tests/queries/0_stateless/02883_named_collections_override.sh b/tests/queries/0_stateless/02883_named_collections_override.sh index 915ce28022622..371574f1f2427 100755 --- a/tests/queries/0_stateless/02883_named_collections_override.sh +++ b/tests/queries/0_stateless/02883_named_collections_override.sh @@ -15,78 +15,53 @@ DROP NAMED COLLECTION IF EXISTS $u2; DROP NAMED COLLECTION IF EXISTS $u3; CREATE NAMED COLLECTION IF NOT EXISTS $u1 AS - url = 'http://127.0.0.1:8123?query=select+1' NOT OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1' NOT OVERRIDABLE, format = 'RawBLOB' OVERRIDABLE; CREATE NAMED COLLECTION IF NOT EXISTS $u2 AS - url = 'http://127.0.0.1:8123?query=select+1', + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1', format = 'RawBLOB'; -SET allow_named_collection_override_by_default=1; -SELECT 'allow_named_collection_override_by_default=1 u1'; +SELECT 'u1'; SELECT * FROM url($u1); SELECT * FROM url($u1, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url($u1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM url($u1, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); -- { serverError BAD_ARGUMENTS } SELECT * FROM url($u1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=1 u2'; +SELECT 'u2'; SELECT * FROM url($u2); SELECT * FROM url($u2, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url($u2, url='http://127.0.0.1:8123?query=select+2'); +SELECT * FROM url($u2, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); SELECT * FROM url($u2, format='CSV'); -SET allow_named_collection_override_by_default=0; -SELECT 'allow_named_collection_override_by_default=0 u1'; -SELECT * FROM url($u1); -SELECT * FROM url($u1, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=0 u2'; -SELECT * FROM url($u2); -SELECT * FROM url($u2, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u2, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u2, format='CSV'); -- { serverError BAD_ARGUMENTS } - SELECT 'Test ALTER'; CREATE NAMED COLLECTION IF NOT EXISTS $u3 AS - url = 'http://127.0.0.1:8123?query=select+1' NOT OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1' NOT OVERRIDABLE, format = 'RawBLOB' OVERRIDABLE; ALTER NAMED COLLECTION $u3 SET - url = 'http://127.0.0.1:8123?query=select+2' OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2' OVERRIDABLE, format = 'RawBLOB' NOT OVERRIDABLE; SELECT * FROM url($u3); -SELECT * FROM url($u3, url='http://127.0.0.1:8123?query=select+1'); +SELECT * FROM url($u3, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1'); SELECT * FROM url($u3, format='CSV'); -- { serverError BAD_ARGUMENTS } DROP NAMED COLLECTION IF EXISTS $u1; DROP NAMED COLLECTION IF EXISTS $u2; -DROP NAMED COLLECTION IF EXISTS $u2; +DROP NAMED COLLECTION IF EXISTS $u3; SELECT 'Test XML collections'; -SET allow_named_collection_override_by_default=1; -SELECT 'allow_named_collection_override_by_default=1 url_override1'; +SELECT 'url_override1'; SELECT * FROM url(url_override1); SELECT * FROM url(url_override1, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url(url_override1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM url(url_override1, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); -- { serverError BAD_ARGUMENTS } SELECT * FROM url(url_override1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=1 url_override2'; +SELECT 'url_override2'; SELECT * FROM url(url_override2); SELECT * FROM url(url_override2, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url(url_override2, url='http://127.0.0.1:8123?query=select+2'); +SELECT * FROM url(url_override2, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); SELECT * FROM url(url_override2, format='CSV'); -SET allow_named_collection_override_by_default=0; -SELECT 'allow_named_collection_override_by_default=0 url_override1'; -SELECT * FROM url(url_override1); -SELECT * FROM url(url_override1, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=0 url_override2'; -SELECT * FROM url(url_override2); -SELECT * FROM url(url_override2, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override2, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override2, format='CSV'); -- { serverError BAD_ARGUMENTS } " diff --git a/tests/queries/0_stateless/04665_nats_credentials_named_collection.sql b/tests/queries/0_stateless/04665_nats_credentials_named_collection.sql index 3a701974b3067..5c3d1a812b4e3 100644 --- a/tests/queries/0_stateless/04665_nats_credentials_named_collection.sql +++ b/tests/queries/0_stateless/04665_nats_credentials_named_collection.sql @@ -96,18 +96,6 @@ SETTINGS nats_credential_file = '/var/other.creds', nats_credentials = 'user JWT CREATE TABLE nats_both_in_settings_over_credentials (key UInt64) ENGINE = NATS(04665_nats_credentials) SETTINGS nats_credentials = 'another user JWT and seed', nats_credential_file = '/var/nats.creds'; -- { serverError BAD_ARGUMENTS } --- An inline credential already stored in the collection is a same-key override, so both query --- spellings must honor the global override policy. -SET allow_named_collection_override_by_default = 0; - -CREATE TABLE nats_credentials_override_disabled (key UInt64) -ENGINE = NATS(04665_nats_credentials, nats_credentials = 'another user JWT and seed'); -- { serverError BAD_ARGUMENTS } - -CREATE TABLE nats_credentials_in_settings_override_disabled (key UInt64) ENGINE = NATS(04665_nats_credentials) -SETTINGS nats_credentials = 'another user JWT and seed'; -- { serverError BAD_ARGUMENTS } - -SET allow_named_collection_override_by_default = 1; - DROP NAMED COLLECTION 04665_nats_credential_file; DROP NAMED COLLECTION 04665_nats_credentials; DROP NAMED COLLECTION 04665_nats_both; diff --git a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql index 6c6549b72bde1..41d85b89955cb 100644 --- a/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql +++ b/tests/queries/0_stateless/04891_nats_credential_file_path_restriction.sql @@ -213,25 +213,3 @@ SETTINGS nats_credentials = 'user JWT and seed'; -- { serverError BAD_ARGUMENTS -- The locked path is still usable as it is stored in the collection. CREATE TABLE nats_locked_config_file (key UInt64) ENGINE = NATS(nats_config_locked_credentials); -- { serverError CANNOT_CONNECT_NATS } - --- Passing the contents is the only way to supply these credentials from SQL, so it is not treated as --- a brand-new key: it stays usable when overrides are forbidden by default, where the operator states --- the permission with `overridable="false"` instead. An unrelated key is still a new key there. -SET allow_named_collection_override_by_default = 0; - -CREATE TABLE nats_credentials_over_config_file_no_override_by_default (key UInt64) -ENGINE = NATS(nats_config_credentials, nats_credentials = 'user JWT and seed'); -- { serverError CANNOT_CONNECT_NATS } - -CREATE TABLE nats_credentials_over_locked_config_file_no_override_by_default (key UInt64) -ENGINE = NATS(nats_config_locked_credentials, nats_credentials = 'user JWT and seed'); -- { serverError BAD_ARGUMENTS } - --- The `SETTINGS` spelling follows the same new-key policy as the engine-argument spelling, --- even though this collection has neither credential key. -CREATE TABLE nats_credentials_in_settings_over_config_collection_without_credentials_no_override_by_default (key UInt64) -ENGINE = NATS(nats_config_no_credentials) -SETTINGS nats_credentials = 'user JWT and seed'; -- { serverError BAD_ARGUMENTS } - -CREATE TABLE nats_subjects_over_config_collection_no_override_by_default (key UInt64) -ENGINE = NATS(nats_config_credentials, nats_subjects = 'other'); -- { serverError BAD_ARGUMENTS } - -SET allow_named_collection_override_by_default = 1; diff --git a/tests/queries/0_stateless/05296_named_collection_override_access.reference b/tests/queries/0_stateless/05296_named_collection_override_access.reference new file mode 100644 index 0000000000000..f829e536949e4 --- /dev/null +++ b/tests/queries/0_stateless/05296_named_collection_override_access.reference @@ -0,0 +1,11 @@ +Usage and additional keys need no secrets privilege +1 +1 +1 +1 +The secrets privilege is scoped to the collection and inherited from roles +2 +1 +1 +Dictionary sources can add keys +42 diff --git a/tests/queries/0_stateless/05296_named_collection_override_access.sh b/tests/queries/0_stateless/05296_named_collection_override_access.sh new file mode 100755 index 0000000000000..a1845cfcbec4a --- /dev/null +++ b/tests/queries/0_stateless/05296_named_collection_override_access.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_url" +remote_nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_remote" +header_nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_headers" +user="${CLICKHOUSE_TEST_UNIQUE_NAME}_user" +role="${CLICKHOUSE_TEST_UNIQUE_NAME}_role" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP DICTIONARY IF EXISTS override_access_dict; + DROP TABLE IF EXISTS override_access_source; + DROP USER IF EXISTS $user; + DROP ROLE IF EXISTS $role; + DROP NAMED COLLECTION IF EXISTS $nc; + DROP NAMED COLLECTION IF EXISTS $remote_nc; + DROP NAMED COLLECTION IF EXISTS $header_nc; + " +} +trap cleanup EXIT +cleanup + +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE NAMED COLLECTION $nc AS + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', + format = 'TSV' OVERRIDABLE; + CREATE NAMED COLLECTION $remote_nc AS + host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, database = '${CLICKHOUSE_DATABASE}', + user = 'default', password = ''; + CREATE NAMED COLLECTION $header_nc AS + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', format = 'TSV', + \`headers.header.name\` = 'Accept', \`headers.header.value\` = 'text/plain'; + CREATE USER $user; + CREATE ROLE $role; + GRANT $role TO $user; + GRANT URL, REMOTE, CREATE TEMPORARY TABLE ON *.* TO $user; + GRANT NAMED COLLECTION ON $nc TO $user; + GRANT NAMED COLLECTION ON $remote_nc TO $user; + GRANT NAMED COLLECTION ON $header_nc TO $user; +" + +echo 'Usage and additional keys need no secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + SELECT * FROM url($nc); + SELECT * FROM url($nc, structure = 'value UInt64'); + SELECT * FROM url($nc, structure = 'value UInt8', structure = 'value UInt64'); + SELECT * FROM url($nc, structure = 'value UInt64', headers('Accept' = 'text/plain')); + SELECT * FROM url($nc, url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+2', structure = 'value UInt64'); -- { serverError ACCESS_DENIED } + SELECT * FROM url($nc, format = 'CSV', structure = 'value UInt64'); -- { serverError ACCESS_DENIED } + SELECT * FROM url($header_nc, structure = 'value UInt64', headers('Accept' = 'text/csv')); -- { serverError ACCESS_DENIED } + SELECT * FROM url($header_nc, structure = 'value UInt64', \`headers.header[0].value\` = 'text/csv'); -- { serverError ACCESS_DENIED } + + SELECT * FROM remote($remote_nc, host = '127.0.0.1', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM remote($remote_nc, hostname = '127.0.0.1', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM remote($remote_nc, addresses_expr = '127.0.0.1:1', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM remote($remote_nc, db = 'system', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM remote($remote_nc, username = 'other', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM remote($remote_nc, database = numbers(1)); -- { serverError ACCESS_DENIED } +" + +echo 'The secrets privilege is scoped to the collection and inherited from roles' +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $nc TO $role" +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + SET format_display_secrets_in_show_and_select = 0; + SELECT * FROM url($nc, url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+2', structure = 'value UInt64'); + SELECT * FROM url($nc, format = 'CSV', structure = 'value UInt64'); + SELECT * FROM remote($remote_nc, addresses_expr = '127.0.0.1:1', table = 'one'); -- { serverError ACCESS_DENIED } + SELECT * FROM url($header_nc, structure = 'value UInt64', headers('Accept' = 'text/csv')); -- { serverError ACCESS_DENIED } +" + +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $header_nc TO $role" +${CLICKHOUSE_CLIENT} --user "$user" --query " + SELECT * FROM url($header_nc, structure = 'value UInt64', headers('Accept' = 'text/csv')); +" + +${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc SET format = 'TSV' NOT OVERRIDABLE" +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + SELECT * FROM url($nc, format = 'CSV', structure = 'value UInt64'); -- { serverError BAD_ARGUMENTS } +" +${CLICKHOUSE_CLIENT} --query "REVOKE SHOW NAMED COLLECTIONS SECRETS ON $nc FROM $role" +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + SELECT * FROM url($nc, url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+2', structure = 'value UInt64'); -- { serverError ACCESS_DENIED } +" + +echo 'Dictionary sources can add keys' +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE TABLE override_access_source (id UInt64, value UInt64) ENGINE = Memory; + INSERT INTO override_access_source VALUES (1, 42); + CREATE DICTIONARY override_access_dict (id UInt64, value UInt64) + PRIMARY KEY id SOURCE(CLICKHOUSE(NAME $remote_nc TABLE 'override_access_source')) + LAYOUT(FLAT()) LIFETIME(0); + SELECT dictGet('override_access_dict', 'value', toUInt64(1)); +" diff --git a/tests/queries/0_stateless/05297_named_collection_override_access_sources.reference b/tests/queries/0_stateless/05297_named_collection_override_access_sources.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05297_named_collection_override_access_sources.sh b/tests/queries/0_stateless/05297_named_collection_override_access_sources.sh new file mode 100755 index 0000000000000..90108638c3add --- /dev/null +++ b/tests/queries/0_stateless/05297_named_collection_override_access_sources.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Requires the external database and message queue engines omitted from the fast build. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}" +user="${nc}_user" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP TABLE IF EXISTS override_access_source; + DROP TABLE IF EXISTS override_access_queue; + DROP USER IF EXISTS $user; + DROP NAMED COLLECTION IF EXISTS ${nc}_db; + DROP NAMED COLLECTION IF EXISTS ${nc}_mysql_settings; + DROP NAMED COLLECTION IF EXISTS ${nc}_pg_settings; + DROP NAMED COLLECTION IF EXISTS ${nc}_s3; + DROP NAMED COLLECTION IF EXISTS ${nc}_azure; + DROP NAMED COLLECTION IF EXISTS ${nc}_nats; + DROP NAMED COLLECTION IF EXISTS ${nc}_kafka; + DROP NAMED COLLECTION IF EXISTS ${nc}_rabbitmq; + " +} +trap cleanup EXIT +cleanup + +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE USER $user; + GRANT SOURCES, CREATE TEMPORARY TABLE ON *.* TO $user; + GRANT CREATE TABLE, SELECT, INSERT, BACKUP ON ${CLICKHOUSE_DATABASE}.* TO $user; + GRANT NAMED COLLECTION ON * TO $user; + CREATE TABLE override_access_source (value UInt64) ENGINE = Memory; + CREATE NAMED COLLECTION ${nc}_db AS + host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table'; + CREATE NAMED COLLECTION ${nc}_mysql_settings AS + host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', + connection_pool_size = 2; + CREATE NAMED COLLECTION ${nc}_pg_settings AS + host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', + postgresql_connection_pool_size = 2; + CREATE NAMED COLLECTION ${nc}_s3 AS + url = 'http://127.0.0.1:1/bucket/data', access_key_id = 'key', secret_access_key = 'secret'; + CREATE NAMED COLLECTION ${nc}_azure AS + storage_account_url = 'http://127.0.0.1:1/account', account_name = 'account', account_key = 'secret', + container = 'container', blob_path = 'data'; + CREATE NAMED COLLECTION ${nc}_nats AS + nats_url = '127.0.0.1:1', nats_subjects = 'subject', nats_format = 'JSONEachRow', + nats_credentials = 'secret'; + CREATE NAMED COLLECTION ${nc}_kafka AS + kafka_broker_list = '127.0.0.1:1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow'; + CREATE NAMED COLLECTION ${nc}_rabbitmq AS + rabbitmq_host_port = '127.0.0.1:1', rabbitmq_exchange_name = 'exchange', rabbitmq_format = 'JSONEachRow'; +" + +while IFS= read -r query; do + [ -n "$query" ] || continue + if error=$(${CLICKHOUSE_CLIENT} --user "$user" --query "$query" 2>&1); then + echo "Expected the secrets privilege to be required: $query" + exit 1 + fi + if [[ "$error" != *"SHOW NAMED COLLECTIONS SECRETS ON"* ]]; then + echo "$error" + exit 1 + fi +done <&1); then + echo 'Expected the locked host to reject its alias' + exit 1 +fi +if [[ "$error" != *"Override not allowed for 'host'"* ]]; then + echo "$error" + exit 1 +fi diff --git a/tests/queries/0_stateless/05315_named_collection_generated_args_access.reference b/tests/queries/0_stateless/05315_named_collection_generated_args_access.reference new file mode 100644 index 0000000000000..35310d48730e9 --- /dev/null +++ b/tests/queries/0_stateless/05315_named_collection_generated_args_access.reference @@ -0,0 +1,20 @@ +Inferred format and structure replace a stored auto value without the secrets privilege +1 +3 +3 +3 +3 +A stored auto value that is not overridable stays locked +BAD_ARGUMENTS +Resolving a stored relative s3 url against s3_base needs the secrets privilege +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +Resolving a stored relative url against url_base needs the secrets privilege +ACCESS_DENIED +ACCESS_DENIED +With the privilege the relative urls are resolved +3 +3 +1 +1 diff --git a/tests/queries/0_stateless/05315_named_collection_generated_args_access.sh b/tests/queries/0_stateless/05315_named_collection_generated_args_access.sh new file mode 100755 index 0000000000000..4a7b68e3e3552 --- /dev/null +++ b/tests/queries/0_stateless/05315_named_collection_generated_args_access.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: Depends on S3 (minio) +# Tag no-replicated-database: named collections are server-global, not database-scoped + +# ClickHouse appends inferred `format` and `structure` values to the arguments of named collection engines and table +# functions and parses them again. Replacing a stored `'auto'` value this way needs no secrets privilege. +# Resolving a relative stored `url` against `s3_base` or `url_base` replaces the stored `url`, so it needs the privilege. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +BUCKET_URL="http://localhost:11111/test" +FILE="${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}" +user="${nc}_user" +table="${CLICKHOUSE_DATABASE}.${nc}" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP TABLE IF EXISTS ${table}_url; + DROP TABLE IF EXISTS ${table}_s3; + DROP TABLE IF EXISTS ${table}_url_relative; + DROP USER IF EXISTS $user; + DROP NAMED COLLECTION IF EXISTS ${nc}_url_auto; + DROP NAMED COLLECTION IF EXISTS ${nc}_s3_auto; + DROP NAMED COLLECTION IF EXISTS ${nc}_s3_relative; + DROP NAMED COLLECTION IF EXISTS ${nc}_url_relative; + DROP NAMED COLLECTION IF EXISTS ${nc}_locked; + " +} +trap cleanup EXIT +cleanup + +${CLICKHOUSE_CLIENT} -q "INSERT INTO FUNCTION s3('${BUCKET_URL}/${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SELECT number, concat('str_', toString(number)) FROM numbers(3) SETTINGS s3_truncate_on_insert = 1" + +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE NAMED COLLECTION ${nc}_url_auto AS + url = '${BUCKET_URL}/${FILE}', format = 'auto', structure = 'auto'; + CREATE NAMED COLLECTION ${nc}_s3_auto AS + url = '${BUCKET_URL}/${FILE}', access_key_id = 'test', secret_access_key = 'testtest', format = 'auto', structure = 'auto'; + CREATE NAMED COLLECTION ${nc}_s3_relative AS + url = '${FILE}', access_key_id = 'test', secret_access_key = 'testtest', format = 'TSV'; + CREATE NAMED COLLECTION ${nc}_url_relative AS + url = '?query=SELECT+1', format = 'TSV'; + CREATE NAMED COLLECTION ${nc}_locked AS + url = '${BUCKET_URL}/${FILE}', format = 'auto' NOT OVERRIDABLE; + CREATE USER $user; + GRANT S3, URL ON *.* TO $user; + GRANT CREATE TABLE, SELECT ON ${CLICKHOUSE_DATABASE}.* TO $user; + GRANT NAMED COLLECTION ON * TO $user; +" + +echo 'Inferred format and structure replace a stored auto value without the secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + CREATE TABLE ${table}_url ENGINE = URL(${nc}_url_auto); +" +${CLICKHOUSE_CLIENT} --user "$user" --query "SHOW CREATE TABLE ${table}_url FORMAT TabSeparatedRaw" | grep -c "format = 'TSV'" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM ${table}_url" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM s3Cluster('test_cluster_two_shards_localhost', ${nc}_s3_auto)" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM s3(${nc}_s3_auto)" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM url(${nc}_url_auto)" + +echo 'A stored auto value that is not overridable stays locked' +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM url(${nc}_locked, format = 'CSV')" 2>&1 | grep -o 'BAD_ARGUMENTS' | head -1 + +echo 'Resolving a stored relative s3 url against s3_base needs the secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --query "SET s3_base = '${BUCKET_URL}/'; CREATE TABLE ${table}_s3 (n UInt32, s String) ENGINE = S3(${nc}_s3_relative)" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM s3(${nc}_s3_relative, structure = 'n UInt32, s String') SETTINGS s3_base = '${BUCKET_URL}/'" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM s3Cluster('test_cluster_two_shards_localhost', ${nc}_s3_relative, structure = 'n UInt32, s String') SETTINGS s3_base = '${BUCKET_URL}/'" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 + +echo 'Resolving a stored relative url against url_base needs the secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM url(${nc}_url_relative, structure = 'x UInt8') SETTINGS url_base = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/'" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 +${CLICKHOUSE_CLIENT} --user "$user" --query "SET url_base = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/'; CREATE TABLE ${table}_url_relative (x UInt8) ENGINE = URL(${nc}_url_relative)" 2>&1 | grep -o 'ACCESS_DENIED' | head -1 + +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON ${nc}_s3_relative TO $user" +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON ${nc}_url_relative TO $user" + +echo 'With the privilege the relative urls are resolved' +${CLICKHOUSE_CLIENT} --user "$user" --query "SET s3_base = '${BUCKET_URL}/'; CREATE TABLE ${table}_s3 (n UInt32, s String) ENGINE = S3(${nc}_s3_relative)" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM ${table}_s3" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM s3(${nc}_s3_relative, structure = 'n UInt32, s String') SETTINGS s3_base = '${BUCKET_URL}/'" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM url(${nc}_url_relative, structure = 'x UInt8') SETTINGS url_base = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/'" +${CLICKHOUSE_CLIENT} --user "$user" --query "SET url_base = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}/'; CREATE TABLE ${table}_url_relative (x UInt8) ENGINE = URL(${nc}_url_relative)" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM ${table}_url_relative" diff --git a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.reference b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.reference new file mode 100644 index 0000000000000..95c94bb2c8039 --- /dev/null +++ b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.reference @@ -0,0 +1,16 @@ +Overriding a stored key in a dictionary source requires the secrets privilege +Adding a missing key needs no secrets privilege +100 +With the secrets privilege the override is allowed +200 +200 +A NOT OVERRIDABLE key cannot be overridden even with the secrets privilege +A dictionary created with an override by a privileged user loads in the background +200 +200 +200 +A stored table whose override became NOT OVERRIDABLE is rejected when it is attached +Override not allowed for 'connection_pool_size' +BAD_ARGUMENTS +Override not allowed for 'http_method' +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh new file mode 100755 index 0000000000000..2039d59fd0bdb --- /dev/null +++ b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: requires the MySQL engine omitted from the fast build. +# Tag no-replicated-database: named collections are server-global, and `DETACH TABLE` is not allowed in a `Replicated` database. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_dict" +nc_load="${CLICKHOUSE_TEST_UNIQUE_NAME}_load" +nc_mysql="${CLICKHOUSE_TEST_UNIQUE_NAME}_mysql" +nc_url="${CLICKHOUSE_TEST_UNIQUE_NAME}_url" +user="${CLICKHOUSE_TEST_UNIQUE_NAME}_user" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP DICTIONARY IF EXISTS dict_override; + DROP DICTIONARY IF EXISTS dict_add_key; + DROP DICTIONARY IF EXISTS dict_alias; + DROP DICTIONARY IF EXISTS dict_locked; + DROP DICTIONARY IF EXISTS dict_load; + DROP TABLE IF EXISTS table_settings_override; + DROP TABLE IF EXISTS table_alias_override; + DROP TABLE IF EXISTS dict_source_a; + DROP TABLE IF EXISTS dict_source_b; + DROP USER IF EXISTS $user; + DROP NAMED COLLECTION IF EXISTS $nc; + DROP NAMED COLLECTION IF EXISTS $nc_load; + DROP NAMED COLLECTION IF EXISTS $nc_mysql; + DROP NAMED COLLECTION IF EXISTS $nc_url; + " +} +trap cleanup EXIT +cleanup + +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE TABLE dict_source_a (id UInt64, value UInt64) ENGINE = Memory; + CREATE TABLE dict_source_b (id UInt64, value UInt64) ENGINE = Memory; + INSERT INTO dict_source_a VALUES (1, 100); + INSERT INTO dict_source_b VALUES (1, 200); + + CREATE NAMED COLLECTION $nc AS + host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', + db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; + CREATE NAMED COLLECTION $nc_load AS + host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', + db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; + + CREATE USER $user; + GRANT SOURCES ON *.* TO $user; + GRANT CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${CLICKHOUSE_DATABASE}.* TO $user; + GRANT NAMED COLLECTION ON $nc TO $user; +" + +echo 'Overriding a stored key in a dictionary source requires the secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + CREATE DICTIONARY dict_override (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError ACCESS_DENIED } + CREATE DICTIONARY dict_alias (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc DATABASE '${CLICKHOUSE_DATABASE}' TABLE 'dict_source_a')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError ACCESS_DENIED } +" + +echo 'Adding a missing key needs no secrets privilege' +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + CREATE DICTIONARY dict_add_key (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc WHERE 'id = 1')) LAYOUT(FLAT()) LIFETIME(0); + SELECT dictGet('dict_add_key', 'value', toUInt64(1)); +" + +echo 'With the secrets privilege the override is allowed' +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $nc TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + CREATE DICTIONARY dict_override (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); + SELECT dictGet('dict_override', 'value', toUInt64(1)); + CREATE DICTIONARY dict_alias (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc DATABASE '${CLICKHOUSE_DATABASE}' TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); + SELECT dictGet('dict_alias', 'value', toUInt64(1)); +" + +echo 'A NOT OVERRIDABLE key cannot be overridden even with the secrets privilege' +${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc SET table = 'dict_source_a' NOT OVERRIDABLE" +${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " + CREATE DICTIONARY dict_locked (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError BAD_ARGUMENTS } +" + +echo 'A dictionary created with an override by a privileged user loads in the background' +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE DICTIONARY dict_load (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(NAME $nc_load TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); + SELECT dictGet('dict_load', 'value', toUInt64(1)); + SYSTEM RELOAD DICTIONARY dict_load; + SELECT dictGet('dict_load', 'value', toUInt64(1)); + DETACH DICTIONARY dict_load; + ATTACH DICTIONARY dict_load; + SELECT dictGet('dict_load', 'value', toUInt64(1)); +" + +echo 'A stored table whose override became NOT OVERRIDABLE is rejected when it is attached' +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE NAMED COLLECTION $nc_mysql AS + host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', + connection_pool_size = 2; + CREATE NAMED COLLECTION $nc_url AS + url = 'http://127.0.0.1:1/data', format = 'TSV', http_method = 'POST'; + CREATE TABLE table_settings_override (value UInt64) ENGINE = MySQL($nc_mysql) SETTINGS connection_pool_size = 1; + CREATE TABLE table_alias_override (value UInt64) ENGINE = URL($nc_url, method = 'PUT'); +" + +# A table that overrides a key which is locked afterwards cannot be attached: the lock is checked on every load. +function attach_is_rejected() +{ + local table=$1 + local key=$2 + ${CLICKHOUSE_CLIENT} --query "DETACH TABLE $table" + if error=$(${CLICKHOUSE_CLIENT} --query "ATTACH TABLE $table" 2>&1); then + echo "Expected the attach of $table to be rejected" + exit 1 + fi + echo "$error" | grep -o "Override not allowed for '$key'" | head -1 + echo "$error" | grep -o 'BAD_ARGUMENTS' | head -1 +} + +${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 NOT OVERRIDABLE" +attach_is_rejected table_settings_override connection_pool_size +${CLICKHOUSE_CLIENT} --multiquery --query " + ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 OVERRIDABLE; + ATTACH TABLE table_settings_override; + ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' NOT OVERRIDABLE; +" +attach_is_rejected table_alias_override http_method +${CLICKHOUSE_CLIENT} --multiquery --query " + ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' OVERRIDABLE; + ATTACH TABLE table_alias_override; +" diff --git a/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.reference b/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.reference new file mode 100644 index 0000000000000..3608758e888e5 --- /dev/null +++ b/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.reference @@ -0,0 +1,6 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +addition OK +granted OK +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.sh b/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.sh new file mode 100755 index 0000000000000..b00421e4dd6ab --- /dev/null +++ b/tests/queries/0_stateless/05317_named_collection_kafka_nested_override_access.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: requires the Kafka engine omitted from the fast build. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}" +nc_locked="${nc}_locked" +user="${nc}_user" +table="${nc}_table" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP TABLE IF EXISTS ${table}; + DROP USER IF EXISTS $user; + DROP NAMED COLLECTION IF EXISTS ${nc}; + DROP NAMED COLLECTION IF EXISTS ${nc_locked}; + " +} +trap cleanup EXIT +cleanup + +# Flattened `SETTINGS` names overwrite the librdkafka properties that are set from nested collection keys. +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE USER $user; + GRANT SOURCES ON *.* TO $user; + GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO $user; + GRANT NAMED COLLECTION ON * TO $user; + CREATE NAMED COLLECTION ${nc} AS + kafka_broker_list = '127.0.0.1:2', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', + \`kafka.security_protocol\` = 'SASL_SSL', + \`kafka.consumer.sasl_username\` = 'user', + \`kafka.producer.compression_codec\` = 'zstd'; + CREATE NAMED COLLECTION ${nc_locked} AS + kafka_broker_list = '127.0.0.1:2', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', + \`kafka.security_protocol\` = 'SASL_SSL' NOT OVERRIDABLE; +" + +function expect_error() +{ + local code="$1" + local query="$2" + if error=$(${CLICKHOUSE_CLIENT} --user "$user" --query "$query" 2>&1); then + echo "Expected $code: $query" + exit 1 + fi + echo "$error" | { grep -o "$code" || true; } | head -1 +} + +# Without the privilege, a flattened setting cannot replace a stored nested key. +expect_error ACCESS_DENIED "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc}) SETTINGS kafka_security_protocol = 'PLAINTEXT'" +expect_error ACCESS_DENIED "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc}) SETTINGS kafka_sasl_username = 'x'" +expect_error ACCESS_DENIED "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc}) SETTINGS kafka_compression_codec = 'gzip'" + +# A flattened setting without a stored nested equivalent is an addition. +${CLICKHOUSE_CLIENT} --user "$user" --query "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc}) SETTINGS kafka_sasl_mechanism = 'PLAIN'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${table}" +echo 'addition OK' + +# With the privilege, the override is allowed. +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON ${nc} TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc}) SETTINGS kafka_security_protocol = 'PLAINTEXT'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${table}" +echo 'granted OK' + +# A grant cannot bypass the explicit lock of a nested key. +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON ${nc_locked} TO $user" +expect_error BAD_ARGUMENTS "CREATE TABLE ${table} (value String) ENGINE = Kafka(${nc_locked}) SETTINGS kafka_security_protocol = 'PLAINTEXT'" diff --git a/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.reference b/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.reference new file mode 100644 index 0000000000000..cb9fa6353fd6b --- /dev/null +++ b/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.reference @@ -0,0 +1,8 @@ +Replacing a stored setting without the secrets privilege +secrets privilege required +Adding a setting that the collection does not store +check passed +Replacing a stored setting with the secrets privilege +check passed +Replacing a stored setting that is not overridable +override not allowed diff --git a/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.sh b/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.sh new file mode 100755 index 0000000000000..6d1321cbadd60 --- /dev/null +++ b/tests/queries/0_stateless/05318_named_collection_postgresql_function_settings_access.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: requires the PostgreSQL engine omitted from the fast build. + +# The `SETTINGS` clause of the `postgresql` table function replaces the settings stored in a named collection, +# so it needs `SHOW NAMED COLLECTIONS SECRETS` and respects `NOT OVERRIDABLE`. The check runs before the connection, +# so the unreachable server only produces a connection error when the check passes. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +nc="${CLICKHOUSE_TEST_UNIQUE_NAME}" +user="${nc}_user" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --multiquery --query " + DROP USER IF EXISTS $user; + DROP NAMED COLLECTION IF EXISTS $nc; + " +} +trap cleanup EXIT +cleanup + +${CLICKHOUSE_CLIENT} --multiquery --query " + CREATE USER $user; + GRANT POSTGRES, CREATE TEMPORARY TABLE ON *.* TO $user; + GRANT NAMED COLLECTION ON $nc TO $user; + CREATE NAMED COLLECTION $nc AS + host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', + postgresql_connection_pool_size = 2; +" + +# Classifies the error of the query: the secrets privilege is missing, the key is locked, +# or the check passed and the query only reached the unreachable server. +function error_code() +{ + local error + if error=$(${CLICKHOUSE_CLIENT} --user "$user" --query "$1" 2>&1); then + echo "Expected an error: $1" + exit 1 + fi + if [[ "$error" == *"SHOW NAMED COLLECTIONS SECRETS ON"* ]]; then + echo 'secrets privilege required' + elif [[ "$error" == *"Override not allowed for 'postgresql_connection_pool_size'"* ]]; then + echo 'override not allowed' + elif [[ "$error" == *"POSTGRESQL_CONNECTION_FAILURE"* ]]; then + echo 'check passed' + else + echo "$error" + fi +} + +echo 'Replacing a stored setting without the secrets privilege' +error_code "SELECT * FROM postgresql($nc, SETTINGS postgresql_connection_pool_size = 1)" + +echo 'Adding a setting that the collection does not store' +error_code "SELECT * FROM postgresql($nc, SETTINGS postgresql_connection_pool_wait_timeout = 1000)" + +echo 'Replacing a stored setting with the secrets privilege' +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $nc TO $user" +error_code "SELECT * FROM postgresql($nc, SETTINGS postgresql_connection_pool_size = 1)" + +echo 'Replacing a stored setting that is not overridable' +${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc SET postgresql_connection_pool_size = 2 NOT OVERRIDABLE" +error_code "SELECT * FROM postgresql($nc, SETTINGS postgresql_connection_pool_size = 1)" From c6e751ea8008f2f2493ed45d4f963f8474678385 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 10:51:07 +0000 Subject: [PATCH 17/81] Backport #125760 to 26.8: Allow vertical merge into non-wide parts --- src/IO/PackedFilesWriter.cpp | 12 +++ src/IO/PackedFilesWriter.h | 4 + .../MergeTree/DataPartStorageOnDiskPacked.cpp | 7 ++ src/Storages/MergeTree/MergeTask.cpp | 2 - ...05394_vertical_merge_packed_part.reference | 13 +++ .../05394_vertical_merge_packed_part.sql | 100 ++++++++++++++++++ ...l_merge_packed_part_empty_result.reference | 3 + ...ertical_merge_packed_part_empty_result.sql | 46 ++++++++ 8 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05394_vertical_merge_packed_part.reference create mode 100644 tests/queries/0_stateless/05394_vertical_merge_packed_part.sql create mode 100644 tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.reference create mode 100644 tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.sql diff --git a/src/IO/PackedFilesWriter.cpp b/src/IO/PackedFilesWriter.cpp index c15e17d11e428..eef44f8ef7c00 100644 --- a/src/IO/PackedFilesWriter.cpp +++ b/src/IO/PackedFilesWriter.cpp @@ -44,6 +44,18 @@ PackedFilesWriter::writeFile(const String & file_name) return std::make_unique(file_name, it->second); } +std::unique_ptr +PackedFilesWriter::rewriteFile(const String & file_name, const WriteSettings & settings) +{ + if (!write_settings && settings != WriteSettings{}) + write_settings = settings; + + /// A buffer of the replaced file may still be alive, but it owns its data and does not affect the new file. + auto & data = written_files[file_name]; + data = std::make_shared(); + return std::make_unique(file_name, data); +} + void PackedFilesWriter::moveFile(const String & from_name, const String & to_name) { metadata_changes.emplace_back(MetadataChange::MOVE, from_name, to_name); diff --git a/src/IO/PackedFilesWriter.h b/src/IO/PackedFilesWriter.h index 834e93c94e893..2e60029c16c9a 100644 --- a/src/IO/PackedFilesWriter.h +++ b/src/IO/PackedFilesWriter.h @@ -39,11 +39,15 @@ class PackedFilesWriter /// Creates memory buffer for the data of file /// and returns fake WriteBufferFromFileBase. + /// Throws if the file is already written. OutBufferPtr writeFile(const String & file_name); /// The same as above, but also updated settings to write file with archive. OutBufferPtr writeFile(const String & file_name, const WriteSettings & settings); + /// The same as above, but replaces the file if it is already written, like rewriting a file on a disk. + OutBufferPtr rewriteFile(const String & file_name, const WriteSettings & settings); + /// Common operations with files which modify only @files map. void moveFile(const String & from_name, const String & to_name); void replaceFile(const String & from_name, const String & to_name); diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp index 6ce082f9f90ab..d6f8debe5bf2d 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp @@ -421,7 +421,14 @@ std::unique_ptr DataPartStorageOnDiskPacked::writeFile( } if (writer) + { + /// Rewrite an already written file as a disk does. For example, when a vertical merge writes no rows, + /// the writers of different subcolumns of `Nested` create the file with their shared offsets each. + if (mode == WriteMode::Rewrite) + return writer->rewriteFile(name, settings); + return writer->writeFile(name, settings); + } throw Exception(ErrorCodes::NOT_INITIALIZED, "Cannot write file {} because writer is not initialized", name); } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index fc33ad1cc3a83..df64dd5f8ce48 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -3539,8 +3539,6 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm } if (global_ctx->future_part->part_format.part_type != MergeTreeDataPartType::Wide) return MergeAlgorithm::Horizontal; - if (global_ctx->future_part->part_format.storage_type != MergeTreeDataPartStorageType::Full) - return MergeAlgorithm::Horizontal; if (global_ctx->cleanup) return MergeAlgorithm::Horizontal; diff --git a/tests/queries/0_stateless/05394_vertical_merge_packed_part.reference b/tests/queries/0_stateless/05394_vertical_merge_packed_part.reference new file mode 100644 index 0000000000000..185b5808eb7b2 --- /dev/null +++ b/tests/queries/0_stateless/05394_vertical_merge_packed_part.reference @@ -0,0 +1,13 @@ +before merge +Wide Packed 3 +after merge +Wide Packed 1 +Vertical Wide +1 +1 +1 +1 +after attach +Wide Packed 1 +1 +1 diff --git a/tests/queries/0_stateless/05394_vertical_merge_packed_part.sql b/tests/queries/0_stateless/05394_vertical_merge_packed_part.sql new file mode 100644 index 0000000000000..68bfc425efbb6 --- /dev/null +++ b/tests/queries/0_stateless/05394_vertical_merge_packed_part.sql @@ -0,0 +1,100 @@ +-- Vertical merge into a `Wide` part with `Packed` storage. Every gathered column is written by its own +-- writer into the same in-memory archive, which rejects a second write of the same file, so the shared +-- offsets of `Nested` must be written only once. Skip indexes over gathered columns are built by the +-- per-column writers, while an index over several columns is built in the horizontal stage. +-- The result is compared with a horizontal merge of the same data into a `Full` part. + +SET flatten_nested = 1; + +DROP TABLE IF EXISTS t_vertical_packed; +DROP TABLE IF EXISTS t_horizontal_full; + +CREATE TABLE t_vertical_packed +( + id UInt64, + s String, + lc LowCardinality(String), + arr Array(UInt64), + n Nested(a UInt64, b String), + m Map(String, UInt64), + sparse UInt64, + INDEX idx_s s TYPE bloom_filter GRANULARITY 1, + INDEX idx_sum id + sparse TYPE minmax GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id +SETTINGS + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = '100G', + enable_vertical_merge_algorithm = 1, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1; + +CREATE TABLE t_horizontal_full AS t_vertical_packed +ENGINE = MergeTree ORDER BY id +SETTINGS + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_vertical_merge_algorithm = 0; + +SYSTEM STOP MERGES t_vertical_packed; + +-- Interleaved keys, so that the merge takes consecutive rows from different source parts. +INSERT INTO t_vertical_packed SELECT + number * 3 AS id, 'str_' || toString(id), toString(id % 17), range(id % 5), + range(id % 4), arrayMap(x -> toString(x), range(id % 4)), map('k' || toString(id % 3), id), if(id % 100 = 0, id, 0) +FROM numbers(300); + +INSERT INTO t_vertical_packed SELECT + number * 3 + 1 AS id, 'str_' || toString(id), toString(id % 17), range(id % 5), + range(id % 4), arrayMap(x -> toString(x), range(id % 4)), map('k' || toString(id % 3), id), if(id % 100 = 0, id, 0) +FROM numbers(300); + +INSERT INTO t_vertical_packed SELECT + number * 3 + 2 AS id, 'str_' || toString(id), toString(id % 17), range(id % 5), + range(id % 4), arrayMap(x -> toString(x), range(id % 4)), map('k' || toString(id % 3), id), if(id % 100 = 0, id, 0) +FROM numbers(300); + +INSERT INTO t_horizontal_full SELECT * FROM t_vertical_packed; + +SELECT 'before merge'; +SELECT part_type, part_storage_type, count() FROM system.parts +WHERE database = currentDatabase() AND table = 't_vertical_packed' AND active +GROUP BY part_type, part_storage_type; + +SYSTEM START MERGES t_vertical_packed; +OPTIMIZE TABLE t_vertical_packed FINAL; +OPTIMIZE TABLE t_horizontal_full FINAL; + +SELECT 'after merge'; +SELECT part_type, part_storage_type, count() FROM system.parts +WHERE database = currentDatabase() AND table = 't_vertical_packed' AND active +GROUP BY part_type, part_storage_type; + +SYSTEM FLUSH LOGS part_log; + +SELECT DISTINCT merge_algorithm, part_type FROM system.part_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND database = currentDatabase() AND table = 't_vertical_packed' AND event_type = 'MergeParts'; + +SELECT (SELECT count(), sum(cityHash64(*)) FROM t_vertical_packed) = (SELECT count(), sum(cityHash64(*)) FROM t_horizontal_full); +SELECT count() FROM t_vertical_packed WHERE s = 'str_300' SETTINGS force_data_skipping_indices = 'idx_s'; +SELECT count() FROM t_vertical_packed WHERE id + sparse = 600 SETTINGS force_data_skipping_indices = 'idx_sum'; + +CHECK TABLE t_vertical_packed SETTINGS check_query_single_value_result = 1; + +-- The part must also be readable after it is loaded from the archive on disk. +DETACH TABLE t_vertical_packed; +ATTACH TABLE t_vertical_packed; + +SELECT 'after attach'; +SELECT part_type, part_storage_type, count() FROM system.parts +WHERE database = currentDatabase() AND table = 't_vertical_packed' AND active +GROUP BY part_type, part_storage_type; + +SELECT (SELECT count(), sum(cityHash64(*)) FROM t_vertical_packed) = (SELECT count(), sum(cityHash64(*)) FROM t_horizontal_full); +SELECT count() FROM t_vertical_packed WHERE s = 'str_300' SETTINGS force_data_skipping_indices = 'idx_s'; + +DROP TABLE t_vertical_packed; +DROP TABLE t_horizontal_full; diff --git a/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.reference b/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.reference new file mode 100644 index 0000000000000..f60994f38991c --- /dev/null +++ b/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.reference @@ -0,0 +1,3 @@ +Wide Packed 2 +0 +Vertical Wide 0 0 diff --git a/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.sql b/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.sql new file mode 100644 index 0000000000000..937a5c442d405 --- /dev/null +++ b/tests/queries/0_stateless/05395_vertical_merge_packed_part_empty_result.sql @@ -0,0 +1,46 @@ +-- Vertical merge into a `Wide` part with `Packed` storage when all rows collapse and the result is empty. +-- Without rows the writers skip the final mark, but they still create the streams of their columns, so the +-- shared offsets of `Nested` have to be registered anyway. Otherwise the writer of the next subcolumn +-- creates the same file in the archive again, which the archive rejects. + +SET flatten_nested = 1; + +DROP TABLE IF EXISTS t_vertical_packed_empty; + +CREATE TABLE t_vertical_packed_empty +( + id UInt64, + sign Int8, + s String, + n Nested(a UInt64, b String, c Array(UInt64)) +) +ENGINE = CollapsingMergeTree(sign) ORDER BY id +SETTINGS + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0, + min_bytes_for_full_part_storage = '100G', + enable_vertical_merge_algorithm = 1, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1; + +SYSTEM STOP MERGES t_vertical_packed_empty; + +INSERT INTO t_vertical_packed_empty VALUES (1, 1, 'x', [1, 2], ['a', 'b'], [[1], [2, 3]]); +INSERT INTO t_vertical_packed_empty VALUES (1, -1, 'x', [1, 2], ['a', 'b'], [[1], [2, 3]]); + +SELECT part_type, part_storage_type, count() FROM system.parts +WHERE database = currentDatabase() AND table = 't_vertical_packed_empty' AND active +GROUP BY part_type, part_storage_type; + +SYSTEM START MERGES t_vertical_packed_empty; +OPTIMIZE TABLE t_vertical_packed_empty FINAL; + +SELECT count() FROM t_vertical_packed_empty; + +SYSTEM FLUSH LOGS part_log; + +SELECT DISTINCT merge_algorithm, part_type, rows, error FROM system.part_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND database = currentDatabase() AND table = 't_vertical_packed_empty' AND event_type = 'MergeParts'; + +DROP TABLE t_vertical_packed_empty; From 4917aa8ffbb7438fa4a09c0c2f9020fd71a053fb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 10:54:29 +0000 Subject: [PATCH 18/81] Backport #122756 to 26.8: Fix `CANNOT_READ_ALL_DATA` in subcolumn size estimation for an empty Wide part --- .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- ...2_empty_wide_part_subcolumn_size.reference | 32 +++++++ .../05292_empty_wide_part_subcolumn_size.sql | 96 +++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.reference create mode 100644 tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.sql diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index cd168f35f355b..d0e9660b18c82 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -237,7 +237,7 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( ColumnSize MergeTreeDataPartWide::calculateSubcolumnSize(const String & subcolumn_name) const { ColumnSize size; - if (checksums.empty()) + if (checksums.empty() || isEmpty()) return size; for (const auto & stream : getListOfStreamsForColumn(getColumn(subcolumn_name))) diff --git a/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.reference b/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.reference new file mode 100644 index 0000000000000..0e3771402a92f --- /dev/null +++ b/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.reference @@ -0,0 +1,32 @@ +t_empty_compact 1 Compact Full 0 +t_empty_compact 2 Compact Full 1 +t_empty_compact 3 Compact Full 1 +t_empty_packed 1 Wide Packed 0 +t_empty_packed 2 Wide Packed 1 +t_empty_packed 3 Compact Packed 1 +t_empty_ttl 1 Wide Full 0 +t_empty_ttl 2 Wide Full 1 +t_empty_ttl 3 Compact Full 1 +t_empty_wide 1 Wide Full 0 +t_empty_wide 2 Wide Full 1 +t_empty_wide 3 Compact Full 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +2 diff --git a/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.sql b/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.sql new file mode 100644 index 0000000000000..261b21b4e5a90 --- /dev/null +++ b/tests/queries/0_stateless/05292_empty_wide_part_subcolumn_size.sql @@ -0,0 +1,96 @@ +-- Random settings limits: optimize_functions_to_subcolumns=(1, 1); optimize_move_to_prewhere=(1, 1); query_plan_optimize_prewhere=(1, 1); allow_calculating_subcolumns_sizes_for_merge_tree_reading=(1, 1) + +-- An active part without rows, left here by a mutation that deleted all its rows or by a TTL drop, must not break +-- the subcolumn size estimate of the PREWHERE optimization, for Wide (full and packed storage) and Compact parts. + +DROP TABLE IF EXISTS t_empty_wide; +DROP TABLE IF EXISTS t_empty_packed; +DROP TABLE IF EXISTS t_empty_compact; +DROP TABLE IF EXISTS t_empty_ttl; + +CREATE TABLE t_empty_wide +( + p UInt8, + m Map(LowCardinality(String), String), + mv Map(String, LowCardinality(String)), + t Tuple(a LowCardinality(String), b UInt64), + v Variant(String, UInt64), + d Dynamic, + j JSON +) +ENGINE = MergeTree PARTITION BY p ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0, remove_empty_parts = 0; + +CREATE TABLE t_empty_packed AS t_empty_wide ENGINE = MergeTree PARTITION BY p ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 1000000000, remove_empty_parts = 0; + +CREATE TABLE t_empty_compact AS t_empty_wide ENGINE = MergeTree PARTITION BY p ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 1000000000, min_bytes_for_full_part_storage = 0, remove_empty_parts = 0; + +CREATE TABLE t_empty_ttl (p UInt8, ts DateTime, m Map(LowCardinality(String), String)) +ENGINE = MergeTree PARTITION BY p ORDER BY tuple() TTL ts +SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0, remove_empty_parts = 0, + ttl_only_drop_parts = 1, merge_with_ttl_timeout = 0; + +INSERT INTO t_empty_wide VALUES (1, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +ALTER TABLE t_empty_wide DELETE WHERE p = 1 SETTINGS mutations_sync = 2; +INSERT INTO t_empty_wide VALUES (2, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +ALTER TABLE t_empty_wide MODIFY SETTING min_bytes_for_wide_part = 1000000000; +INSERT INTO t_empty_wide VALUES (3, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); + +INSERT INTO t_empty_packed VALUES (1, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +ALTER TABLE t_empty_packed DELETE WHERE p = 1 SETTINGS mutations_sync = 2; +INSERT INTO t_empty_packed VALUES (2, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +ALTER TABLE t_empty_packed MODIFY SETTING min_bytes_for_wide_part = 1000000000; +INSERT INTO t_empty_packed VALUES (3, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); + +INSERT INTO t_empty_compact VALUES (1, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +ALTER TABLE t_empty_compact DELETE WHERE p = 1 SETTINGS mutations_sync = 2; +INSERT INTO t_empty_compact VALUES (2, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); +INSERT INTO t_empty_compact VALUES (3, {'k': 'v'}, {'k': 'v'}, ('v', 1), 'v', 'v', '{"a": "v"}'); + +INSERT INTO t_empty_ttl VALUES (1, '2000-01-01', {'k': 'v'}); +-- Keep the part written by the TTL drop instead of merging it again. +OPTIMIZE TABLE t_empty_ttl PARTITION 1 FINAL SETTINGS optimize_skip_merged_partitions = 1; +INSERT INTO t_empty_ttl VALUES (2, '2100-01-01', {'k': 'v'}); +ALTER TABLE t_empty_ttl MODIFY SETTING min_bytes_for_wide_part = 1000000000; +INSERT INTO t_empty_ttl VALUES (3, '2100-01-01', {'k': 'v'}); + +SELECT table, partition, part_type, part_storage_type, rows FROM system.parts +WHERE database = currentDatabase() AND table LIKE 't_empty_%' AND active +ORDER BY table, partition; + +-- Only a Compact part is left after partition pruning. +SELECT count() FROM t_empty_wide WHERE p = 3 AND m['k'] = 'v'; +SELECT count() FROM t_empty_wide WHERE p = 3 AND mv['k'] = 'v'; +SELECT count() FROM t_empty_wide WHERE p = 3 AND t.a = 'v'; +SELECT count() FROM t_empty_wide WHERE p = 3 AND v.String = 'v'; +SELECT count() FROM t_empty_wide WHERE p = 3 AND d.String = 'v'; +SELECT count() FROM t_empty_wide WHERE p = 3 AND j.a::String = 'v'; + +SELECT count() FROM t_empty_packed WHERE p = 3 AND m['k'] = 'v'; +SELECT count() FROM t_empty_packed WHERE p = 3 AND mv['k'] = 'v'; +SELECT count() FROM t_empty_packed WHERE p = 3 AND t.a = 'v'; +SELECT count() FROM t_empty_packed WHERE p = 3 AND v.String = 'v'; +SELECT count() FROM t_empty_packed WHERE p = 3 AND d.String = 'v'; +SELECT count() FROM t_empty_packed WHERE p = 3 AND j.a::String = 'v'; + +SELECT count() FROM t_empty_compact WHERE p = 3 AND m['k'] = 'v'; +SELECT count() FROM t_empty_compact WHERE p = 3 AND mv['k'] = 'v'; +SELECT count() FROM t_empty_compact WHERE p = 3 AND t.a = 'v'; +SELECT count() FROM t_empty_compact WHERE p = 3 AND v.String = 'v'; +SELECT count() FROM t_empty_compact WHERE p = 3 AND d.String = 'v'; +SELECT count() FROM t_empty_compact WHERE p = 3 AND j.a::String = 'v'; + +SELECT count() FROM t_empty_ttl WHERE p = 3 AND m['k'] = 'v'; + +-- Automatic parallel replicas. +SELECT count() FROM t_empty_wide WHERE m['k'] = 'v' +SETTINGS enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 2, parallel_replicas_local_plan = 1, + automatic_parallel_replicas_min_bytes_per_replica = 0, max_parallel_replicas = 2, + cluster_for_parallel_replicas = 'parallel_replicas', parallel_replicas_for_non_replicated_merge_tree = 1; + +DROP TABLE t_empty_wide; +DROP TABLE t_empty_packed; +DROP TABLE t_empty_compact; +DROP TABLE t_empty_ttl; From 5baea8e808dea3df1dc4a6ca93f592f449bb7a9f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 15:27:18 +0000 Subject: [PATCH 19/81] Backport #120887 to 26.8: Reject deeply nested JSON documents instead of overflowing the stack --- src/Formats/JSONExtractTree.cpp | 188 +++++++++++++----- src/Formats/JSONExtractTree.h | 5 +- src/Functions/FunctionsJSON.cpp | 23 ++- .../05218_json_type_deep_recursion.reference | 8 + .../05218_json_type_deep_recursion.sql | 51 +++++ 5 files changed, 224 insertions(+), 51 deletions(-) create mode 100644 tests/queries/0_stateless/05218_json_type_deep_recursion.reference create mode 100644 tests/queries/0_stateless/05218_json_type_deep_recursion.sql diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index bc8dde8e72454..4860888b0f712 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -4,6 +4,7 @@ #include #include +#include #if USE_SIMDJSON #include #endif @@ -69,22 +70,22 @@ namespace ErrorCodes } template -void jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings) +bool jsonElementToStringImpl(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth, size_t depth) { if (element.isInt64()) { writeIntText(element.getInt64(), buf); - return; + return true; } if (element.isUInt64()) { writeIntText(element.getUInt64(), buf); - return; + return true; } if (element.isDouble()) { writeFloatText(element.getDouble(), buf); - return; + return true; } if (element.isBool()) { @@ -92,28 +93,42 @@ void jsonElementToString(const typename JSONParser::Element & element, WriteBuff writeCString("true", buf); else writeCString("false", buf); - return; + return true; } if (element.isString()) { writeJSONString(element.getString(), buf, format_settings); - return; + return true; + } + if (element.isArray() || element.isObject()) + { + if (max_depth && depth > max_depth) + { + writeCString("...", buf); + return false; + } + + /// The parsers build the document iteratively, so its nesting depth is bounded only by the + /// input. Checked in the nested branches only, to keep the scalar path free. + checkStackSize(); } if (element.isArray()) { + bool complete = true; writeChar('[', buf); bool need_comma = false; for (auto value : element.getArray()) { if (std::exchange(need_comma, true)) writeChar(',', buf); - jsonElementToString(value, buf, format_settings); + complete &= jsonElementToStringImpl(value, buf, format_settings, max_depth, depth + 1); } writeChar(']', buf); - return; + return complete; } if (element.isObject()) { + bool complete = true; writeChar('{', buf); bool need_comma = false; for (auto [key, value] : element.getObject()) @@ -122,16 +137,21 @@ void jsonElementToString(const typename JSONParser::Element & element, WriteBuff writeChar(',', buf); writeJSONString(key, buf, format_settings); writeChar(':', buf); - jsonElementToString(value, buf, format_settings); + complete &= jsonElementToStringImpl(value, buf, format_settings, max_depth, depth + 1); } writeChar('}', buf); - return; + return complete; } if (element.isNull()) - { writeCString("null", buf); - return; - } + + return true; +} + +template +bool jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth) +{ + return jsonElementToStringImpl(element, buf, format_settings, max_depth, 1); } template @@ -276,6 +296,29 @@ String jsonElementToString(const typename JSONParser::Element & element, const F return buf.str(); } +/// A message quotes untrusted input, so producing one must not recurse deeply or grow huge. +constexpr size_t MAX_DEPTH_IN_ERROR_MESSAGE = 64; + +template +String jsonElementToStringForError(const typename JSONParser::Element & element, const FormatSettings & format_settings) +{ + WriteBufferFromOwnString buf; + jsonElementToString(element, buf, format_settings, MAX_DEPTH_IN_ERROR_MESSAGE); + return buf.str(); +} + +/// Writes the element as a column's text value. A value cannot be elided like a message, so one +/// nested deeper than the limit is rejected instead. +template +bool jsonElementToStringValue(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, String & error) +{ + if (jsonElementToString(element, buf, format_settings, format_settings.json.max_depth)) + return true; + + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; +} + template class NumericNode : public JSONExtractTreeNode { @@ -313,7 +356,7 @@ class NumericNode : public JSONExtractTreeNode if (!tryGetNumericValueFromJSONElement(value, element, /*convert_bool_to_number=*/ true, insert_settings.allow_type_conversion, insert_settings.no_int_truncation_from_double, format_settings.precise_float_parsing, error)) { if (error.empty()) - error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToString(element, format_settings)); + error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToStringForError(element, format_settings)); return false; } @@ -370,7 +413,7 @@ class LowCardinalityNumericNode : public NumericNode if (!tryGetNumericValueFromJSONElement(value, element, /*convert_bool_to_number=*/ true, insert_settings.allow_type_conversion, insert_settings.no_int_truncation_from_double, format_settings.precise_float_parsing, error)) { if (error.empty()) - error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToString(element, format_settings)); + error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToStringForError(element, format_settings)); return false; } @@ -415,9 +458,16 @@ class StringNode : public JSONExtractTreeNode auto & col_str = assert_cast(column); auto & chars = col_str.getChars(); + size_t prev_chars_size = chars.size(); + bool written = false; { WriteBufferFromVector buf(chars, AppendModeTag(), format_settings.json_max_string_column_growth_step); - jsonElementToString(element, buf, format_settings); + written = jsonElementToStringValue(element, buf, format_settings, error); + } + if (!written) + { + chars.resize_assume_reserved(prev_chars_size); + return false; } col_str.getOffsets().push_back(chars.size()); } @@ -461,7 +511,11 @@ class LowCardinalityStringNode : public JSONExtractTreeNode if (!insert_settings.allow_type_conversion) return false; - auto value = jsonElementToString(element, format_settings); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + auto value = buf.str(); assert_cast(column).insertData(value.data(), value.size()); } else @@ -505,7 +559,11 @@ class FixedStringNode : public JSONExtractTreeNode { if (!insert_settings.allow_type_conversion) return false; - return checkValueSizeAndInsert(column, jsonElementToString(element, format_settings), error); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + return checkValueSizeAndInsert(column, buf.str(), error); } return checkValueSizeAndInsert(column, element.getString(), error); } @@ -556,7 +614,11 @@ class LowCardinalityFixedStringNode : public JSONExtractTreeNode { if (!insert_settings.allow_type_conversion) return false; - return checkValueSizeAndInsert(column, jsonElementToString(element, format_settings), error); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + return checkValueSizeAndInsert(column, buf.str(), error); } return checkValueSizeAndInsert(column, element.getString(), error); } @@ -610,7 +672,7 @@ class UUIDNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -655,7 +717,7 @@ class LowCardinalityUUIDNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -694,7 +756,7 @@ class DateNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read Date value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Date value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -792,7 +854,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi } else { - error = fmt::format("cannot read DateTime value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read DateTime value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -862,7 +924,7 @@ class TimeNode : public JSONExtractTreeNode } else { - error = fmt::format("cannot read Time value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Time value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -943,7 +1005,7 @@ class DecimalNode : public JSONExtractTreeNode } default: { - error = fmt::format("cannot read Decimal value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Decimal value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -1044,7 +1106,7 @@ class DateTime64Node : public JSONExtractTreeNode, public TimezoneMi } break; default: - error = fmt::format("cannot read DateTime64 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read DateTime64 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -1129,7 +1191,7 @@ class Time64Node : public JSONExtractTreeNode value.value = element.getInt64(); break; default: - error = fmt::format("cannot read Time64 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Time64 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -1223,7 +1285,7 @@ class EnumNode : public JSONExtractTreeNode return true; } - error = fmt::format("cannot read Enum value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Enum value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1253,7 +1315,7 @@ class IPv4Node : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read IPv4 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read IPv4 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1295,7 +1357,7 @@ class IPv6Node : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read IPv6 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read IPv6 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1406,7 +1468,7 @@ class ArrayNode : public JSONExtractTreeNode if (!element.isArray()) { - error = fmt::format("cannot read Array value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Array value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1568,7 +1630,7 @@ class TupleNode : public JSONExtractTreeNode return were_valid_elements; } - error = fmt::format("cannot read Tuple value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Tuple value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1599,7 +1661,7 @@ class MapNode : public JSONExtractTreeNode if (!element.isObject()) { - error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1705,7 +1767,7 @@ class VariantNode : public JSONExtractTreeNode } } - error = fmt::format("cannot read Variant value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Variant value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1780,13 +1842,17 @@ class DynamicNode : public JSONExtractTreeNode /// We couldn't insert element into current variants (or skipped it intentionally), /// infer ClickHouse type for this element and add it as a new variant. - auto element_type = removeNullable(elementToDataType(element, format_settings)); + auto inferred_type = elementToDataType(element, format_settings, error); + if (!inferred_type) + return false; + + auto element_type = removeNullable(inferred_type); if (!checkIfTypeIsComplete(element_type)) { error = fmt::format( "Cannot infer the type of JSON element {}, because it contains only nulls. To use String type for elements with incomplete " "type, enable setting input_format_json_infer_incomplete_types_as_strings", - jsonElementToString(element, format_settings)); + jsonElementToStringForError(element, format_settings)); return false; } @@ -1815,10 +1881,14 @@ class DynamicNode : public JSONExtractTreeNode return true; } - DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings) const + /// Returns nullptr with the reason in `error` if the element is nested too deep. + DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings, String & error) const { JSONInferenceInfo json_inference_info; - auto type = elementToDataTypeImpl(element, format_settings, json_inference_info); + auto type = elementToDataTypeImpl(element, format_settings, json_inference_info, 1, error); + if (!type) + return nullptr; + transformFinalInferredJSONTypeIfNeeded(type, format_settings, &json_inference_info); if (format_settings.schema_inference_make_columns_nullable && type->haveSubtypes()) type = makeNullableRecursively(type, format_settings); @@ -1826,8 +1896,18 @@ class DynamicNode : public JSONExtractTreeNode } private: - DataTypePtr elementToDataTypeImpl(const typename JSONParser::Element & element, const FormatSettings & format_settings, JSONInferenceInfo & json_inference_info) const + DataTypePtr elementToDataTypeImpl(const typename JSONParser::Element & element, const FormatSettings & format_settings, JSONInferenceInfo & json_inference_info, size_t depth, String & error) const { + /// Array nesting depth is bounded only by the input and becomes the depth of the inferred + /// type, so the limit also protects the later unguarded recursions over that type (building + /// its name, its extract tree, its column, destroying it). checkStackSize is the backstop. + checkStackSize(); + if (depth > format_settings.json.max_depth) + { + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return nullptr; + } + switch (element.type()) { case ElementType::NULL_VALUE: @@ -1869,7 +1949,12 @@ class DynamicNode : public JSONExtractTreeNode DataTypes types; types.reserve(array.size()); for (auto value : array) - types.push_back(elementToDataTypeImpl(value, format_settings, json_inference_info)); + { + auto value_type = elementToDataTypeImpl(value, format_settings, json_inference_info, depth + 1, error); + if (!value_type) + return nullptr; + types.push_back(std::move(value_type)); + } if (types.empty()) return std::make_shared(std::make_shared()); @@ -1956,7 +2041,7 @@ class ObjectJSONNode : public JSONExtractTreeNode if (!element.isObject()) { - error = fmt::format("Cannot read JSON object from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("Cannot read JSON object from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1968,7 +2053,7 @@ class ObjectJSONNode : public JSONExtractTreeNode /// It's not optimal, but it's a price we pay for faster reading of subcolumns. std::vector> paths_and_values_for_shared_data; /// Temporary Dynamic column that will be used to create and serialize values in shared data. - if (!traverseAndInsert(column_object, element, "", insert_settings, format_settings, paths_and_values_for_shared_data, prev_size, error, true)) + if (!traverseAndInsert(column_object, element, "", insert_settings, format_settings, paths_and_values_for_shared_data, prev_size, error, true, 1)) { /// If there was an error, restore previous state. SerializationObject::restoreColumnObject(column_object, prev_size); @@ -2049,8 +2134,19 @@ class ObjectJSONNode : public JSONExtractTreeNode size_t current_size, String & error, bool is_root, + size_t depth, bool skip_typed_path_check = false) const { + /// Object nesting depth is bounded only by the input; the limit also keeps the memory spent + /// on paths, which every level appends to, from growing quadratically with the depth. + /// checkStackSize is the backstop. + checkStackSize(); + if (depth > format_settings.json.max_depth) + { + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; + } + if (shouldSkipPath(current_path, insert_settings)) return true; @@ -2148,7 +2244,7 @@ class ObjectJSONNode : public JSONExtractTreeNode && typed_path_nodes.contains(path) && !canParseObjectValue(typed_paths_types.at(path)); - if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false, skip_typed)) + if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false, depth + 1, skip_typed)) return false; } @@ -2744,16 +2840,16 @@ std::unique_ptr> buildJSONExtractTree(const Data } #if USE_SIMDJSON -template void jsonElementToString(const SimdJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const SimdJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); #endif #if USE_RAPIDJSON -template void jsonElementToString(const RapidJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const RapidJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); template bool tryGetNumericValueFromJSONElement(Float64 & value, const RapidJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, bool no_int_truncation_from_double, bool precise_float_parsing, String & error); #else -template void jsonElementToString(const DummyJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const DummyJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); template bool tryGetNumericValueFromJSONElement(Float64 & value, const DummyJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, bool no_int_truncation_from_double, bool precise_float_parsing, String & error); template bool tryGetNumericValueFromJSONElement(Int64 & value, const DummyJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, bool no_int_truncation_from_double, bool precise_float_parsing, String & error); diff --git a/src/Formats/JSONExtractTree.h b/src/Formats/JSONExtractTree.h index ca5cd13138c09..531413b44c7e0 100644 --- a/src/Formats/JSONExtractTree.h +++ b/src/Formats/JSONExtractTree.h @@ -49,8 +49,11 @@ class JSONExtractTreeNode template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); +/// Writes the element as JSON text, descending at most `max_depth` levels and writing `...` in place +/// of deeper values. `max_depth` == 0 does not bound the descent, leaving checkStackSize as the only +/// guard. Returns false if anything was elided. template -void jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +bool jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth = 0); template bool tryGetNumericValueFromJSONElement(NumberType & value, const typename JSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, bool no_int_truncation_from_double, bool precise_float_parsing, String & error); diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 3f34c2478fbbe..891d0c8c846ef 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -1163,13 +1163,22 @@ class JSONExtractRawImpl static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view, const FormatSettings & format_settings, String &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view, const FormatSettings & format_settings, String & error) { ColumnString & col_str = assert_cast(dest); auto & chars = col_str.getChars(); + size_t prev_chars_size = chars.size(); + bool written = false; { WriteBufferFromVector buf(chars, AppendModeTag()); - jsonElementToString(element, buf, format_settings); + written = jsonElementToString(element, buf, format_settings, format_settings.json.max_depth); + } + /// A value nested deeper than the limit cannot be rendered whole: a failure to extract it. + if (!written) + { + chars.resize_assume_reserved(prev_chars_size); + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; } col_str.getOffsets().push_back(chars.size()); return true; @@ -1199,7 +1208,11 @@ class JSONExtractArrayRawImpl ColumnArray & col_res = assert_cast(dest); for (auto value : array) - JSONExtractRawImpl::insertResultToColumn(col_res.getData(), value, {}, format_settings, error); + { + /// The result keeps one element per array element, also where one cannot be rendered. + if (!JSONExtractRawImpl::insertResultToColumn(col_res.getData(), value, {}, format_settings, error)) + col_res.getData().insertDefault(); + } col_res.getOffsets().push_back(col_res.getOffsets().back() + array.size()); return true; @@ -1237,7 +1250,9 @@ class JSONExtractKeysAndValuesRawImpl for (const auto & [key, value] : object) { col_key.insertData(key.data(), key.size()); - JSONExtractRawImpl::insertResultToColumn(col_value, value, {}, format_settings, error); + /// Keep one value per key even where the value cannot be rendered. + if (!JSONExtractRawImpl::insertResultToColumn(col_value, value, {}, format_settings, error)) + col_value.insertDefault(); } col_arr.getOffsets().push_back(col_arr.getOffsets().back() + object.size()); diff --git a/tests/queries/0_stateless/05218_json_type_deep_recursion.reference b/tests/queries/0_stateless/05218_json_type_deep_recursion.reference new file mode 100644 index 0000000000000..7d4db6a5b7f06 --- /dev/null +++ b/tests/queries/0_stateless/05218_json_type_deep_recursion.reference @@ -0,0 +1,8 @@ +0 +0 +0 +['1','','2'] +[('a','1'),('b','')] +1 +{"a":{"b":[1,2,3]},"c":"42"} +{"b":[1,2,3]} diff --git a/tests/queries/0_stateless/05218_json_type_deep_recursion.sql b/tests/queries/0_stateless/05218_json_type_deep_recursion.sql new file mode 100644 index 0000000000000..5e25ac261ac43 --- /dev/null +++ b/tests/queries/0_stateless/05218_json_type_deep_recursion.sql @@ -0,0 +1,51 @@ +-- Tags: no-fasttest +-- no-fasttest: needs the RapidJSON parser, which the Fast test build does not include. + +-- Deeply nested JSON documents must be rejected instead of exhausting the thread stack. The +-- RapidJSON parser (allow_simdjson = 0) builds the document iteratively, so it survives any depth +-- and the depth reaches the consumers that walk the parsed document. +SET allow_simdjson = 0; + +-- Each case must have a single possible rejector: which of the limit and the checkStackSize +-- backstop comes first depends on the build, since frame sizes and the share of the stack the +-- backstop allows differ. + +-- Raising the limit above any reachable depth leaves the backstop as the only rejector. Nested +-- objects go through the object traversal, nested arrays through the type inference of a dynamic +-- path value. +SELECT CAST(concat(repeat('{"x":', 300000), '0', repeat('}', 300000)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } +SELECT CAST(concat('{"a":', repeat('[', 300000), repeat(']', 300000), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- Extraction is bounded by the limit, and a value that exceeds it is a failure to extract, which +-- yields the default like any other. All the extractors of a value as text agree on that. +SELECT length(JSONExtract(concat(repeat('[', 300000), repeat(']', 300000)), 'String')); +SELECT length(JSONExtractRaw(concat(repeat('[', 300000), repeat(']', 300000)))); +SELECT length(JSONExtractString(concat(repeat('[', 300000), repeat(']', 300000)))); +SELECT JSONExtractArrayRaw(concat('[1,', repeat('[', 300000), repeat(']', 300000), ',2]')); +SELECT JSONExtractKeysAndValuesRaw(concat('{"a":1,"b":', repeat('[', 300000), repeat(']', 300000), '}')); + +-- Raising the limit above any reachable depth leaves the backstop as the only rejector here too. +SELECT JSONExtractRaw(concat(repeat('[', 300000), repeat(']', 300000))) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- The limit governs in both directions. These depths stay shallow: a document that survives parsing +-- is walked again by consumers whose recursion this setting does not bound. +SELECT CAST(concat(repeat('{"x":', 50), '0', repeat('}', 50)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat(repeat('{"x":', 50), '0', repeat('}', 50)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100; +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100; + +-- A value written as text into a String column is bounded by the limit as well, and exceeding it is +-- reported the same way rather than escaping as an exception. +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON(a String)) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON(a LowCardinality(String))) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } + +-- An element quoted in an error message is rendered shallowly, so the message stays small. +SELECT CAST(concat('{"a":', repeat('[', 200), '1', repeat(']', 200), '}') AS JSON(a UInt32)) FORMAT Null; -- { serverError INCORRECT_DATA } + +-- Exceeding the limit is a parse error, so a too-deep row can be skipped like any other malformed row. +SELECT count() FROM format(JSONEachRow, 'x JSON', concat('{"x":', repeat('{"a":', 50), '0', repeat('}', 50), '}\n{"x":{"a":42}}')) +SETTINGS input_format_json_max_depth = 20, input_format_allow_errors_num = 1; + +-- Shallow documents must still be parsed normally. +SELECT CAST('{"a" : {"b" : [1, 2, 3]}, "c" : "42"}' AS JSON); +SELECT JSONExtractRaw('{"a" : {"b" : [1, 2, 3]}}', 'a'); From 1d4194400422f5bbcad3d41966959265e5c37c42 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 16:46:42 +0000 Subject: [PATCH 20/81] Update autogenerated version to 26.8.28.14 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 797c1fa466218..29446392b0d64 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54540) +SET(VERSION_REVISION 54541) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 28) -SET(VERSION_GITHASH 551899e257e06d9e3c977481944b69695adb3c32) -SET(VERSION_DESCRIBE v26.8.28.1-lts) -SET(VERSION_STRING 26.8.28.1) +SET(VERSION_PATCH 29) +SET(VERSION_GITHASH 3694fe07040369cef78c937311cbaac5f0555484) +SET(VERSION_DESCRIBE v26.8.29.1-lts) +SET(VERSION_STRING 26.8.29.1) # end of autochange From 0733faebaa31ede086432017a15ee16b4a881b8d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 16:46:43 +0000 Subject: [PATCH 21/81] Backport #125900 to 26.8: Fix stack overflow in the CapnProto schema compiler on long chains of declarations --- contrib/capnproto | 2 +- src/Formats/CapnProtoSchema.cpp | 7 ++ ...o_deep_schema_declaration_chains.reference | 17 +++++ ...apnproto_deep_schema_declaration_chains.sh | 75 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference create mode 100755 tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh diff --git a/contrib/capnproto b/contrib/capnproto index e7261205a6bea..7fababf324f50 160000 --- a/contrib/capnproto +++ b/contrib/capnproto @@ -1 +1 @@ -Subproject commit e7261205a6bea770d98e33ac94a35c2cd29e5b19 +Subproject commit 7fababf324f50e6a68be9415294ec2a4c3543208 diff --git a/src/Formats/CapnProtoSchema.cpp b/src/Formats/CapnProtoSchema.cpp index 9bbbf7047d3ba..8183d3a7bde3a 100644 --- a/src/Formats/CapnProtoSchema.cpp +++ b/src/Formats/CapnProtoSchema.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -182,6 +183,9 @@ void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const Str capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaInfo & schema_info) { + /// Bounds the compiler's recursion over chains of declarations, which the text check cannot see. + [[maybe_unused]] static const bool stack_check_installed = (capnp::compiler::setStackCheck(&checkStackSize), true); + checkCapnProtoSchemaNestingDepth(schema_info.schemaDirectory(), schema_info.schemaPath()); capnp::ParsedSchema schema; @@ -197,6 +201,9 @@ capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaIn /// That's not good to determine the type of error by its description, but /// this is the only way to do it here, because kj doesn't specify the type of error. auto description = std::string_view(e.getDescription().cStr()); + if (e.getType() == kj::Exception::Type::OVERLOADED && description.contains("nested too deeply")) + throw Exception(ErrorCodes::CANNOT_PARSE_CAPN_PROTO_SCHEMA, "The CapnProto schema is nested too deeply"); + if (description.contains("No such file or directory") || description.contains("no such directory") || description.contains("no such file")) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot open CapnProto schema, file {} doesn't exists", schema_info.absoluteSchemaPath()); diff --git a/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference new file mode 100644 index 0000000000000..ef3f1a345d487 --- /dev/null +++ b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference @@ -0,0 +1,17 @@ +member_chain +1 +application_chain +1 +nested_suffix_chains +1 +alias_chain +1 +const_chain +1 +struct_chain_with_defaults +1 +enum_annotation_chain +1 +data Int32 +s Tuple(\n f Tuple(\n f Tuple(\n x Int32))) +e Enum8(\'v\' = 0) diff --git a/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh new file mode 100755 index 0000000000000..566c44d4fd8df --- /dev/null +++ b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Long chains of type expression suffixes or of declarations that refer to each other have no deep bracket nesting, +# but the Cap'n Proto schema compiler recurses once per link. They must be rejected, not overflow the stack. + +guard_message_count() { + local output count + output=$(cat) + count=$(grep -c -E 'Code: 622\..*nested too deeply' <<< "$output") || true + echo "$count" + [ "$count" = 1 ] || echo "FAIL: the depth guard did not fire, got: $(head -c 1000 <<< "$output" | tr '\n' ' ')" +} + +SCHEMA_DIR="$(mktemp -d "${CLICKHOUSE_TMP}/capnproto_schemas_XXXXXX")" +SCHEMA_DIR="$(cd "${SCHEMA_DIR}" && pwd)" +trap 'rm -rf "${SCHEMA_DIR}"' EXIT + +python3 - "${SCHEMA_DIR}" <<'PY' +import sys +directory = sys.argv[1] +header = '@0x844f048b15c12dab;\n' +message = 'struct M { data @0 :Int32; }\n' + +def alias_chain(n): + return ''.join(f'using A{i} = A{i + 1};\n' for i in range(n)) + f'using A{n} = Int32;\n' + +def const_chain(n): + return ''.join(f'const c{i} :Int32 = .c{i + 1};\n' for i in range(n)) + f'const c{n} :Int32 = 1;\n' + +def struct_chain(n): + return ''.join(f'struct S{i} {{ f @0 :S{i + 1}; }}\n' for i in range(n)) + f'struct S{n} {{ x @0 :Int32; }}\n' + +# P's default values load every S{i} first, so only walking the chain from P.d0 goes deep. +def struct_chain_with_defaults(n): + return 'struct P { ' + ' '.join(f'd{i} @{i} :S{i} = ();' for i in range(n + 1)) + ' }\n' + struct_chain(n) + +def enum_annotation_chain(n): + return ''.join(f'annotation t{i}(*) :E{i + 1};\nenum E{i} {{ v @0 $t{i}(v); }}\n' for i in range(n)) + f'enum E{n} {{ v @0; }}\n' + +def schema(name, text): + with open(directory + '/' + name + '.capnp', 'w') as f: + f.write(header + text) + +# Long enough to overflow the stack without the guard, and several times the length at which a release build rejects them. +n = 5000 +n_suffix = 20000 +schema('member_chain', 'struct M { data @0 :Int32' + '.a' * n_suffix + '; }\n') +schema('application_chain', 'struct M { data @0 :Int32' + '()' * n_suffix + '; }\n') +schema('nested_suffix_chains', 'struct M { data @0 :' + 'List(' * 10 + 'Int32' + (')' + '.a' * 200) * 10 + '; }\n') +schema('alias_chain', 'struct M { data @0 :A0; }\n' + alias_chain(n)) +schema('const_chain', message + const_chain(n)) +# A struct field costs less stack per link than the chains above. +schema('struct_chain_with_defaults', message + struct_chain_with_defaults(12000)) +schema('enum_annotation_chain', message + enum_annotation_chain(n)) +schema('shallow_chains', 'struct M { data @0 :A0; s @1 :S0; e @2 :E0; }\n' + + alias_chain(2) + const_chain(2) + struct_chain_with_defaults(2) + enum_annotation_chain(2)) +PY + +for name in member_chain application_chain nested_suffix_chains alias_chain const_chain struct_chain_with_defaults enum_annotation_chain +do + echo "${name}" + ${CLICKHOUSE_LOCAL} --logger.console=0 --query " + DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/${name}.capnp:M' + " 2>&1 | guard_message_count +done + +# Short chains of the same shapes are valid schemas. +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/shallow_chains.capnp:M' +" From 1b51ba02098c7ce06922bc85c072dfe666bf3b22 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 18:17:27 +0000 Subject: [PATCH 22/81] Backport #119368 to 26.8: Do not deserialize unexpected packets in TCPHandler --- .../interfaces/specs/NativeProtocol.mdx | 47 +- src/Server/TCPHandler.cpp | 205 ++++--- src/Server/TCPHandler.h | 17 +- ...nexpected_packet_no_deserialization.python | 502 ++++++++++++++++++ ...pected_packet_no_deserialization.reference | 14 + ...75_unexpected_packet_no_deserialization.sh | 16 + 6 files changed, 665 insertions(+), 136 deletions(-) create mode 100644 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python create mode 100644 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.reference create mode 100755 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.sh diff --git a/docs/reference/interfaces/specs/NativeProtocol.mdx b/docs/reference/interfaces/specs/NativeProtocol.mdx index 778b5a0f5c39c..55c8b3fc34479 100644 --- a/docs/reference/interfaces/specs/NativeProtocol.mdx +++ b/docs/reference/interfaces/specs/NativeProtocol.mdx @@ -254,21 +254,23 @@ stateDiagram-v2 HANDSHAKE --> READY: handshake ok READY --> READING_RESPONSE: send Query READING_RESPONSE --> READY: EndOfStream / Exception - READY --> READY: Ping / Pong + READY --> READY: Ping / Pong, TablesStatusRequest / TablesStatusResponse + READING_RESPONSE --> READING_RESPONSE: query input (Data / Scalar / QueryPlan) / Ping / Cancel HANDSHAKE --> Terminated: handshake error + READY --> Terminated: protocol violation / I/O error READING_RESPONSE --> Terminated: protocol violation / I/O error READY --> Terminated: close Terminated --> [*] ``` -The happy path runs straight down — `HANDSHAKE → READY → READING_RESPONSE → READY` — with the `Ping`/`Pong` self-loop and every failure edge funnelling into the single `Terminated` sink. +The happy path runs straight down — `HANDSHAKE → READY → READING_RESPONSE → READY` — with the request/response self-loops in `READY`, the query-input self-loop in `READING_RESPONSE`, and every failure edge funnelling into the single `Terminated` sink. | State | Description | |--------------------|-------------| | `HANDSHAKE` | Initial state after the TCP connection opens. Only [handshake](#handshake-phase) messages are valid. Transitions to `READY` on success or terminates on failure. | -| `READY` | Idle. The client may send [Ping](#ping-phase), [Query](#query-phase), or close. The connection may stay in `READY` indefinitely (subject to `idle_connection_timeout`, see [connection limits](#connection-limits)). A `Data` (2) or `Scalar` (7) packet here is a protocol violation: the server terminates the connection without decoding the packet body, and without sending an `Exception` packet first. Because the body is left unread, the client may observe a TCP reset rather than an orderly close. | -| `READING_RESPONSE` | Entered when the client sends a Query. The client must fully drain the server's response stream before returning to `READY`. The only allowed client→server packet here is Cancel (not specified on this page). | +| `READY` | Idle. The client may send [Ping](#ping-phase), [Query](#query-phase), `TablesStatusRequest` (5), or close. A `TablesStatusRequest` is answered with a `TablesStatusResponse` (9) and leaves the connection in `READY`; its body is not specified on this page. A `Cancel` (3) with no query running is ignored. The connection may stay in `READY` indefinitely (subject to `idle_connection_timeout`, see [connection limits](#connection-limits)). Any other packet here — for example `Hello` (0), `Data` (2), `Scalar` (7), `IgnoredPartUUIDs` (8), or `QueryPlan` (13) — is a protocol violation: the server terminates the connection without decoding the packet body, and without sending an `Exception` packet first. Because the body is left unread, the client may observe a TCP reset rather than an orderly close. See [Protocol violations](#protocol-violations). | +| `READING_RESPONSE` | Entered when the client sends a Query. The query may first need input from the client: the external-table and `input()` blocks, the [INSERT](#insert-phase) row stream, and the [QueryPlan](#queryplan) packet that follows a `Query` with `stage = QueryPlan` are all sent here as `Data` (2), `Scalar` (7), and `QueryPlan` (13) packets, each terminated as described in its phase. While the server is still reading that input the client may also send `Ping` (4); after it, while the server streams the result, only `Cancel` (3, no body) is accepted, plus — during distributed execution — the answers to the server's read-task requests (not specified on this page). `Cancel` is accepted in both parts. The client must fully drain the server's response stream before returning to `READY`. Any other packet is a protocol violation, see [Protocol violations](#protocol-violations). | | Terminated | No longer usable. The client must open a new TCP connection and restart the handshake. | ### Handshake phase {#handshake-phase} @@ -350,6 +352,8 @@ sequenceDiagram participant S as Server C->>S: Query message + C->>S: Scalar packets (0 or more, inter-server) + C->>S: QueryPlan packet (only after a Query with stage = QueryPlan) C->>S: External-table Data packets (0 or more) C->>S: Empty Data marker — end-of-client-data (required) S->>C: Data — header block (N cols, 0 rows) @@ -367,7 +371,7 @@ sequenceDiagram On error at any point the server sends an `Exception` instead of `EndOfStream`, which terminates the query. 1. The client sends [`Query`](#query) with a unique `query_id` (typically a UUID). -2. The client sends any external tables, then the empty Data marker. The empty Data packet has `table_name = ""`, `num_columns = 0`, `num_rows = 0`. The server does not begin executing the query until it receives this marker. An external table may be split over several `Data` packets with the same `table_name`; the first of them binds the table's schema, and every later packet for that name must carry the same column names and types, in the same order, or the server rejects the query with `INCORRECT_DATA` (see [Data](#data)). +2. The client sends any query input, then the empty Data marker. In this order: the `Scalar` (7) packets of an inter-server query, then the [`QueryPlan`](#queryplan) packet if the `Query` had `stage = QueryPlan`, then the external tables. All of them belong to the query's input and must precede the marker: a `Scalar` or `QueryPlan` packet sent after it is a [protocol violation](#protocol-violations). The empty Data packet has `table_name = ""`, `num_columns = 0`, `num_rows = 0`. The server does not begin executing the query until it receives this marker. The server ends the input on any empty block, not only on an empty `Data` packet: a `Scalar` packet whose block has `0` columns is read the same way and also terminates the input, so every packet after it — another `Scalar`, a `QueryPlan`, external tables — is then late. A client must never send a `Scalar` with an empty block. An external table may be split over several `Data` packets with the same `table_name`; the first of them binds the table's schema, and every later packet for that name must carry the same column names and types, in the same order, or the server rejects the query with `INCORRECT_DATA` (see [Data](#data)). 3. The client moves to `READING_RESPONSE` and flushes its write buffer. 4. The client reads response packets in a loop, dispatching by type: @@ -385,7 +389,7 @@ On error at any point the server sends an `Exception` instead of `EndOfStream`, | `Exception` (2) | Decode and return as error. Exit the loop and return to `READY`. | | anything else | Unexpected during the query phase. Terminate the connection. | -On `EndOfStream` or a handled `Exception` the connection returns to `READY`. A protocol violation or I/O error terminates it. +On `EndOfStream` or a handled `Exception` the connection returns to `READY`. A protocol violation or I/O error terminates it — see [Protocol violations](#protocol-violations) for which client packets are violations while the server is still reading the query's input, and what the server does with their bodies. The `num_rows == 0` case trips up new implementations. A zero-row block is a boundary marker or schema header, not an end-of-stream signal. Only `EndOfStream` or `Exception` ends the response. @@ -435,6 +439,31 @@ sequenceDiagram The connection returns to `READY` on `EndOfStream` or a handled `Exception`. Protocol violations and I/O errors terminate it. +A failed `INSERT` is a handled `Exception`, not a violation: if the server rejects the rows (for example a `CHECK` constraint fails) while the client is still streaming, it sends the `Exception` at once and then keeps reading the client's remaining `Data` packets up to the end-of-input terminator, decoding each only far enough to find where it ends, so that the connection stays usable for the next query. A client that reads the `Exception` may either finish its row stream and reuse the connection, or close the socket. + +This applies only when the failure happened between packets, which is the case for a rejected row: the packet carrying it was read in full first. A failure *while decoding* a packet — a column type that does not exist, a malformed value, a memory limit reached mid-block — leaves that packet half-read, and the rest of its body cannot be told apart from the packet that follows it. The server sends the `Exception` and terminates the connection in that case, without reading any further, so the client must not expect to reuse it. Here too the unread bytes mean the client may see a TCP reset instead of an orderly close, and may lose the `Exception` with it. + +### Protocol violations {#protocol-violations} + +A client packet that cannot appear at the current point of the exchange is a protocol violation. The server rejects it as soon as it has read the packet-type code — **it never reads the body**. Nothing sent after the type byte is decoded, and since the body is left unread the client may observe a TCP reset rather than an orderly close. The response depends on the state the connection was in: + +| Connection state | Violating client packets | Server response | +|-------------------------------------------------------|--------------------------|-----------------| +| `READY` (waiting for a `Query`, `Ping`, or `TablesStatusRequest`) | Any packet other than `Query` (1), `Ping` (4), `TablesStatusRequest` (5), and `Cancel` (3), which is ignored here — for example `Hello` (0), `Data` (2), `Scalar` (7), `IgnoredPartUUIDs` (8), or `QueryPlan` (13) | Terminates the connection. No `Exception` packet is sent, because the peer has not necessarily authenticated yet — in the [inter-server secret](#inter-server-secret) mode authentication happens only while the `Query` packet is being read. | +| Reading the query's input (after `Query`, before the end-of-data marker or the end of an `INSERT` row stream) | `Hello` (0), `Query` (1), `TablesStatusRequest` (5), `IgnoredPartUUIDs` (8), and a `QueryPlan` (13) that is out of place — one that does not follow a `Query` with `stage = QueryPlan`, or a second one for the same query | Sends an [`Exception`](#exception) with code `101` (`UNEXPECTED_PACKET_FROM_CLIENT`) and terminates the connection. | +| Executing the query (after its input has ended) | Any packet other than `Cancel` (3), `IgnoredPartUUIDs` (8), and the inter-server read-task replies described in the next row, including a late `QueryPlan` (13) | Terminates the connection without an `Exception` packet, as for an unknown packet-type code. | +| Executing a distributed query, while the server waits for a reply to its own request | Any packet other than `Cancel` (3) and the requested reply: `ReadTaskResponse` (9) after a `ReadTaskRequest`, `MergeTreeReadTaskResponse` (10) after a `MergeTreeReadTaskRequest`, or [`MergeTreeAllRangesAnnouncementResponse`](#mergetreeallrangesannouncementresponse) (14) after a `MergeTreeAllRangesAnnouncement` that expects one. These replies are inter-server only and are accepted only while their request is outstanding; at any other point they fall under the previous row. | Sends an [`Exception`](#exception) with code `101` (`UNEXPECTED_PACKET_FROM_CLIENT`) and terminates the connection. | +| Executing the query (after its input has ended) | `IgnoredPartUUIDs` (8) | Sends an [`Exception`](#exception) with code `101` (`UNEXPECTED_PACKET_FROM_CLIENT`) and terminates the connection, as while reading the query's input. | +| Any state | An unknown packet-type code | Terminates the connection. | + +The `Exception` in the second row is sent only while the query is still live. Once a query has failed, the server keeps reading its remaining input to discard it (see the paragraph above on a failed `INSERT`), and a violating packet arriving during that discard is rejected the same way but silently: the connection is closed without a further `Exception`, so the client sees only the original query's exception, and only if that had already been flushed to it. The same applies to a query run with `run_query_in_background = 1` that finished before reading all of its input: the server sends `EndOfStream` first and then discards the remaining input, and a violation during that discard closes the connection without an `Exception` after `EndOfStream`. + +Because the body is never consumed, the offending packet's bytes are not parsed as later packets either: after rejecting a `Query` packet sent in the middle of an `INSERT`, for example, the server does not go on to interpret the rest of that packet's body as a new packet stream. A client that has sent a violating packet must open a new connection; nothing further can be read from or written to the old one. + +Rejecting the packet before its body also means the body's length does not matter. A client that sends only a type code and then waits will get the response above immediately rather than after `receive_timeout`. + +`IgnoredPartUUIDs` (8) is an obsolete packet: it belonged to the removed setting `allow_experimental_query_deduplication`, and no current server accepts it in any state. Its `Exception` message names the removed setting. + ## Message reference {#message-reference} Fields are listed in wire order. The `Type` column uses: @@ -672,7 +701,7 @@ The wire format is symmetric — both directions include a `table_name` prefix b | table_name | String | universal | External table name. Empty (`""`) is the common case — for the main table, query results, and the INSERT row stream. Empty `table_name` alone is **not** the end-of-data marker (normal INSERT row packets also carry `""`). A non-empty name may repeat across client `Data` packets of one query: the first packet with a given name creates the external table and binds its schema (a packet with columns but `num_rows = 0` counts), and each later packet with that name must declare the same column names and types in the same order — the server compares the block header with the bound schema and answers with an `Exception` (`INCORRECT_DATA`) on a mismatch. | | Block body | — | — | See [Block & column structure](/reference/interfaces/specs/NativeFormat#block-and-column-structure). | -The **end-of-data marker** is a packet whose Block is empty — `0` columns and `0` rows — regardless of `table_name`. The server treats a client `Data` packet as the terminator only when the decoded block is empty (`block.empty()`); a packet with `table_name = ""` and a non-empty block is an ordinary row packet, not a terminator. So an INSERT row stream is a sequence of non-empty `Data` blocks followed by one empty `Data` block that ends it. +The **end-of-data marker** is a packet whose Block is empty — `0` columns and `0` rows — regardless of `table_name`. The server treats a client `Data` packet as the terminator only when the decoded block is empty (`block.empty()`). While reading a query's input it applies the same check to `Scalar` packets, so an empty `Scalar` block is a terminator too; a packet with `table_name = ""` and a non-empty block is an ordinary row packet, not a terminator. So an INSERT row stream is a sequence of non-empty `Data` blocks followed by one empty `Data` block that ends it. The block variants and what they mean are documented under [Block variants](/reference/interfaces/specs/NativeFormat#block-variants). @@ -926,7 +955,7 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th | 5 | TablesStatusRequest | not specified | Table status check | | 6 | KeepAlive | not specified | Connection keepalive | | 7 | Scalar | not specified | Scalar data block | -| 8 | IgnoredPartUUIDs | not specified | Parts to exclude from query | +| 8 | IgnoredPartUUIDs | not specified | Obsolete (query deduplication was removed). Rejected without reading the body in every state — see [Protocol violations](#protocol-violations). | | 9 | ReadTaskResponse | not specified | S3 cluster read response | | 10 | MergeTreeReadTaskResponse | not specified | Parallel read task response | | 11 | SSHChallengeRequest | [SSH auth](#ssh-authentication) | SSH auth challenge request | @@ -960,7 +989,7 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th ### QueryPlan {#queryplan} -`QueryPlan` is an inter-server-only client packet. It follows a [`Query`](#query) whose `stage` is `QueryPlan` (`7`) and contains one serialized plan stream. +`QueryPlan` is an inter-server-only client packet. It follows a [`Query`](#query) whose `stage` is `QueryPlan` (`7`) and contains one serialized plan stream. It is part of the query's input: the initiator sends it after the query's `Scalar` packets and before the external tables and the empty `Data` end-of-data marker (see the [Query phase](#query-phase)). Exactly one `QueryPlan` is sent per such query; a `QueryPlan` that arrives after the end-of-data marker terminates the connection. | # | Field | Type | Gate | Description | |---|---|---|---|---| diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4ede0ac90c07c..27ce8ff4daaac 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -261,6 +261,21 @@ void validateClientInfo(const ClientInfo & session_client_info, const ClientInfo // os_user, quota_key, client_trace_context can be different. } } + +/// Reject a packet that cannot appear at this point of the protocol without reading its payload. +/// `UNEXPECTED_PACKET_FROM_CLIENT` closes the connection, so the payload is never needed - and +/// deserializing it would expose a large amount of code (data types, serializations and aggregate +/// function states in a `Data` packet) to a peer that, in interserver mode, has not authenticated yet. +/// The unread payload is also why the connection cannot be reused: the leftover input starts +/// mid-packet, so `runImpl` does not try to skip it for this error code. +[[noreturn]] void throwUnexpectedPacket(UInt64 packet_type) +{ + throw Exception( + DB::ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, + "Unexpected packet {} received from client", + Protocol::Client::toString(packet_type)); +} + struct TurnOffBoolSettingTemporary { bool & setting; @@ -996,7 +1011,21 @@ void TCPHandler::runImpl() sendEndOfStream(*query_state); if (query_state->run_query_in_background && !query_state->read_all_data) - skipData(*query_state); + { + /// `EndOfStream` is already sent, so the client expects nothing more for this query, + /// and an `Exception` sent after it would break the protocol. A protocol violation + /// in the drain (an out-of-place packet, a malformed block) closes the connection silently. + try + { + skipData(*query_state); + } + catch (...) + { + query_state->cancelOut(out); + tryLogCurrentException(log, "Can't skip excessive input packets after the end of a detached query. Close connection."); + return; + } + } } query_state->finalizeOut(out); @@ -1153,8 +1182,11 @@ void TCPHandler::runImpl() std::lock_guard lock(*callback_mutex); /// A query packet is always followed by one or more data packets. - /// If some of those data packets are left, try to skip them. - if (!query_state->read_all_data) + /// If some of those data packets are left, try to skip them - but only from a packet + /// boundary. Skipping from inside a packet parses its remaining bytes as packets of + /// their own, deserializing whatever they say and then blocking until `receive_timeout`. + if (!query_state->read_all_data && query_state->at_packet_boundary + && exception_code != ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT) skipData(*query_state); } catch (...) @@ -1165,9 +1197,10 @@ void TCPHandler::runImpl() } /// We close the connection after an exception if there is something wrong with the connection, - /// otherwise we try to preserve it and reuse for other queries. - if (exception->code() == ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT || exception->code() == ErrorCodes::USER_EXPIRED - || exception->code() == ErrorCodes::TCP_CONNECTION_LIMIT_REACHED) + /// otherwise we try to preserve it and reuse for other queries. A half-read packet counts + /// as wrong: the next query would read the rest of it as a packet of its own. + if (!query_state->at_packet_boundary || exception_code == ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT + || exception_code == ErrorCodes::USER_EXPIRED || exception_code == ErrorCodes::TCP_CONNECTION_LIMIT_REACHED) { LOG_DEBUG(log, "Going to close connection due to exception: {}", exception->message()); query_state->finalizeOut(out); @@ -1225,7 +1258,7 @@ bool TCPHandler::receivePacketsExpectQuery(std::shared_ptr & state) switch (packet_type) { case Protocol::Client::Hello: - processUnexpectedHello(); + throwUnexpectedPacket(packet_type); case Protocol::Client::Data: case Protocol::Client::Scalar: @@ -1238,7 +1271,7 @@ bool TCPHandler::receivePacketsExpectQuery(std::shared_ptr & state) if (is_interserver_mode && !is_interserver_authenticated) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Unexpected data packet received before interserver authentication"); - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Data received from client"); + throwUnexpectedPacket(packet_type); case Protocol::Client::Ping: writeVarUInt(Protocol::Server::Pong, *out); @@ -1298,18 +1331,16 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) } UInt64 packet_type = 0; + /// The type is a varuint, so a client can leave the input mid-packet by truncating it. + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Query: - processUnexpectedQuery(); - case Protocol::Client::Hello: - processUnexpectedHello(); - case Protocol::Client::TablesStatusRequest: - processUnexpectedTablesStatusRequest(); + throwUnexpectedPacket(packet_type); case Protocol::Client::IgnoredPartUUIDs: processObsoleteIgnoredPartUUIDs(); @@ -1319,7 +1350,7 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) { bool empty_block = false; if (state.skipping_data) - empty_block = !processUnexpectedData(state); + empty_block = !skipDataPacket(state); else empty_block = !processData(state, packet_type == Protocol::Client::Scalar); if (empty_block) @@ -1331,12 +1362,14 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) return receiveQueryPlan(state); case Protocol::Client::Ping: + state.at_packet_boundary = true; /// no body writeVarUInt(Protocol::Server::Pong, *out); out->finishChunk(); out->next(); continue; case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return false; // We return false from this function as if no more data received @@ -1781,25 +1814,6 @@ void TCPHandler::processTablesStatusRequest() } -void TCPHandler::processUnexpectedTablesStatusRequest() -{ - /// Consume the same wire prefix as processTablesStatusRequest: on a new-protocol - /// interserver connection the request body is preceded by the authentication hash. -#if USE_SSL - if (is_interserver_mode && client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_TABLES_STATUS) - { - std::string skipped_hash; - readStringBinary(skipped_hash, *in, 32); - } -#endif - - TablesStatusRequest skip_request; - skip_request.read(*in, client_tcp_protocol_version); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet TablesStatusRequest received from client"); -} - - void TCPHandler::sendReadTaskRequest() { writeVarUInt(Protocol::Server::ReadTaskRequest, *out); @@ -2321,23 +2335,6 @@ void TCPHandler::receiveAddendum() } -void TCPHandler::processUnexpectedHello() -{ - UInt64 skip_uint_64 = 0; - String skip_string; - - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readVarUInt(skip_uint_64, *in); - readVarUInt(skip_uint_64, *in); - readVarUInt(skip_uint_64, *in); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Hello received from client"); -} - - void TCPHandler::sendHello() { writeVarUInt(Protocol::Server::Hello, *out); @@ -2441,11 +2438,13 @@ void TCPHandler::sendHello() ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskResponse(QueryState & state) { UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return {}; @@ -2453,6 +2452,7 @@ ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskRes { auto task = std::make_shared(); task->deserialize(*in); + state.at_packet_boundary = true; return task; } @@ -2466,11 +2466,13 @@ ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskRes std::optional TCPHandler::receivePartitionMergeTreeReadTaskResponse(QueryState & state) { UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return {}; @@ -2478,6 +2480,7 @@ std::optional TCPHandler::receivePartitionMergeTreeReadTas { ParallelReadResponse response; response.deserialize(*in, client_parallel_replicas_protocol_version); + state.at_packet_boundary = true; return response; } @@ -2492,16 +2495,22 @@ std::optional TCPHandler::receivePartitionMergeTreeReadTas InitialAllRangesAnnouncementResponse TCPHandler::receiveAllRangesAnnouncementResponse(QueryState & state) { UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return {}; case Protocol::Client::MergeTreeAllRangesAnnouncementResponse: - return InitialAllRangesAnnouncementResponse::deserialize(*in, client_parallel_replicas_protocol_version); + { + auto response = InitialAllRangesAnnouncementResponse::deserialize(*in, client_parallel_replicas_protocol_version); + state.at_packet_boundary = true; + return response; + } default: throw Exception( @@ -2607,7 +2616,6 @@ void TCPHandler::processQuery(std::shared_ptr & state) "Unknown compression state: {}", compression); state->compression = static_cast(compression); - last_block_in.compression = state->compression; readStringBinary(state->query, *in); @@ -2615,6 +2623,11 @@ void TCPHandler::processQuery(std::shared_ptr & state) if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) passed_params = readQueryParameters(*in); + /// The packet is read, so the data packets that always follow it are owed from here on, whether + /// or not the rest of the query is set up successfully. Marking it any later would strand that + /// data for the next query on this connection to read as its first packet. + state->read_all_data = false; + if (is_interserver_mode) { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; @@ -2835,53 +2848,13 @@ void TCPHandler::processQuery(std::shared_ptr & state) std::chrono::milliseconds ms(sleep_after_receiving_query.totalMilliseconds()); std::this_thread::sleep_for(ms); } - - state->read_all_data = false; -} - -void TCPHandler::processUnexpectedQuery() -{ - UInt64 skip_uint_64 = 0; - String skip_string; - - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - - ClientInfo skip_client_info; - if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - skip_client_info.read(*in, client_tcp_protocol_version); - - Settings skip_settings; - auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS - : SettingsWriteFormat::BINARY; - skip_settings.read(*in, settings_format); - - /// The same layout as in `processQuery`: the externally granted roles precede the interserver hash. - if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) - readStringBinary(skip_string, *in, MAX_EXTRA_ROLES_SIZE); - - std::string skip_hash; - bool interserver_secret = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET; - if (interserver_secret) - readStringBinary(skip_hash, *in, 32); - - readVarUInt(skip_uint_64, *in); - - readVarUInt(skip_uint_64, *in); - last_block_in.compression = static_cast(skip_uint_64); - - readStringBinary(skip_string, *in); - - if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) - std::ignore = readQueryParameters(*in); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Query received from client"); } void TCPHandler::processObsoleteIgnoredPartUUIDs() { - /// Reject before reading the peer-controlled payload: this packet only ever arrives pre-query, - /// so the exception closes the connection - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + /// Reject before reading the peer-controlled payload. The error code has to be one that closes + /// the connection, because the unread payload cannot be told apart from the next packet. + throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Received IgnoredPartUUIDs packet, but query deduplication " "(allow_experimental_query_deduplication) is no longer supported. " "Disable the setting on the initiator or finish the cluster upgrade."); @@ -2889,24 +2862,19 @@ void TCPHandler::processObsoleteIgnoredPartUUIDs() bool TCPHandler::receiveQueryPlan(QueryState & state) { - bool unexpected_packet = state.stage != QueryProcessingStage::QueryPlan || state.plan_and_sets || !state.query_context || state.read_all_data; - auto context = unexpected_packet ? Context::getGlobalContextInstance() : state.query_context; - - /// The plan will not be executed when we are draining leftover packets (skipping_data) or when it - /// arrives unexpectedly (deserialized against the global context, which lacks the parallel-replicas - /// coordinator callbacks). Only consume the bytes off the buffer, without building a runnable plan. - if (state.skipping_data || unexpected_packet) - { - QueryPlan::deserialize(*in, context, getBinaryTypeDecodingComplexityLimit(context), /*skip_data=*/true); + /// Rejected while draining a failed query too: consuming it would only buy the reuse of a + /// connection whose query is already lost, at the price of running the plan deserializer on a + /// peer's bytes - which is what `process_query_plan_packet` exists to prevent, and a query + /// rejected for having it off fails late enough that the drain runs. + if (state.skipping_data || state.stage != QueryProcessingStage::QueryPlan || state.plan_and_sets + || !state.query_context || state.read_all_data) + throwUnexpectedPacket(Protocol::Client::QueryPlan); - if (!state.skipping_data && unexpected_packet) - throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet QueryPlan received from client"); - - return true; - } + const auto & context = state.query_context; /// Query plans can be sent by a client here, so guard type decoding with the effective input limit. auto plan_and_sets = QueryPlan::deserialize(*in, context, getBinaryTypeDecodingComplexityLimit(context)); + state.at_packet_boundary = true; LOG_TRACE(log, "Received query plan"); state.plan_and_sets = std::make_shared(std::move(plan_and_sets)); @@ -2923,6 +2891,7 @@ bool TCPHandler::processData(QueryState & state, bool scalar) /// Read one block from the network and write it down Block block = state.block_in->read(); + state.at_packet_boundary = true; if (block.empty()) return false; @@ -2989,21 +2958,25 @@ bool TCPHandler::processData(QueryState & state, bool scalar) } -bool TCPHandler::processUnexpectedData(QueryState & state) +bool TCPHandler::skipDataPacket(QueryState & state) { + /// The client keeps sending the data of a query that has already failed. The block is + /// deserialized only to find where the packet ends, so that the rest of the data can be + /// discarded and the connection reused for the next query. String skip_external_table_name; readStringBinary(skip_external_table_name, *in); std::shared_ptr maybe_compressed_in; - if (last_block_in.compression == Protocol::Compression::Enable) + if (state.compression == Protocol::Compression::Enable) maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else maybe_compressed_in = in; - auto skip_block_in = std::make_shared( - *maybe_compressed_in, client_tcp_protocol_version, getNativeWireFormatSettings(state.query_context)); - bool empty_block = skip_block_in->read().empty(); - return !empty_block; + NativeReader skip_block_in(*maybe_compressed_in, client_tcp_protocol_version, getNativeWireFormatSettings(state.query_context)); + bool has_rows = !skip_block_in.read().empty(); + + state.at_packet_boundary = true; + return has_rows; } @@ -3173,11 +3146,13 @@ void TCPHandler::receivePacketsExpectCancel(QueryState & state) throw NetException(ErrorCodes::ABORTED, "Client has dropped the connection, cancel the query."); UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); break; diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index e9e7eb9a22d55..623082dd66c35 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -118,6 +118,10 @@ struct QueryState /// If true, the data packets will be skipped instead of reading. Used to recover after errors. bool skipping_data = false; + /// Whether the input is positioned at the start of a packet. Cleared before the packet type is + /// read and set again once the packet is fully consumed, so any throw in between leaves it + /// cleared. Input starting mid-packet can only be discarded with the connection. + bool at_packet_boundary = true; bool query_duration_already_logged = false; bool run_query_in_background = false; @@ -151,11 +155,6 @@ struct QueryState }; -struct LastBlockInputParameters -{ - Protocol::Compression compression = Protocol::Compression::Disable; -}; - class TCPHandler : public Poco::Net::TCPServerConnection { public: @@ -259,9 +258,6 @@ class TCPHandler : public Poco::Net::TCPServerConnection /// All the methods which are run inside callbacks are marked with TSA_REQUIRES. std::shared_ptr callback_mutex = std::make_shared(); - /// Last block input parameters are saved to be able to receive unexpected data packet sent after exception. - LastBlockInputParameters last_block_in; - CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection}; /// It is the name of the server that will be sent to the client. @@ -302,11 +298,8 @@ class TCPHandler : public Poco::Net::TCPServerConnection void readTemporaryTables(QueryState & state) TSA_REQUIRES(callback_mutex); void skipData(QueryState & state) TSA_REQUIRES(callback_mutex); + bool skipDataPacket(QueryState & state) TSA_REQUIRES(callback_mutex); - bool processUnexpectedData(QueryState & state); - [[noreturn]] void processUnexpectedQuery(); - [[noreturn]] void processUnexpectedHello(); - [[noreturn]] void processUnexpectedTablesStatusRequest(); /// Reject the obsolete IgnoredPartUUIDs packet (allow_experimental_query_deduplication was removed). [[noreturn]] void processObsoleteIgnoredPartUUIDs(); diff --git a/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python b/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python new file mode 100644 index 0000000000000..a695ecc74133f --- /dev/null +++ b/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# A packet that cannot appear at the current point of the native TCP protocol must be rejected with +# `UNEXPECTED_PACKET_FROM_CLIENT` (101) without reading its payload off the wire. +# +# Deserializing a payload only to throw it away exposes a large amount of code - data type parsing, +# serializations and aggregate function states in a `Data` packet - to a peer that, in interserver +# mode, has not authenticated yet. Every scenario below therefore sends the packet type and nothing +# else: a server that reads the payload waits for the missing bytes until the receive timeout, so +# the socket timeout here fails the test instead of the server answering right away. +# +# An unread payload cannot be told apart from the next packet, so the connection cannot be reused +# either: every scenario keeps the socket open and requires the server to close it. A server that +# drains the leftover input instead parses that payload as a packet stream of its own and blocks. +# +# The client speaks the raw native protocol pinned to revision 54482, so the set of exchanged, +# revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "unexpected packet" + +REVISION = 54482 + +UNEXPECTED_PACKET_FROM_CLIENT = 101 +VIOLATED_CONSTRAINT = 469 + +UNKNOWN_TYPE_NAME = "ThisTypeDoesNotExist" + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt8(s): + return readUInt(s) + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). The server sends its changed settings here. + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings (they are set in the query text) + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack("= 54480) + for _ in range(readVarUInt(s)): + readUInt32(s) + else: + raise RuntimeError("Unknown BlockInfo field {}".format(field_num)) + + +def readKindStack(s): + kind = readUInt8(s) + if kind == 5: # COMBINATION: number of kinds and the kinds themselves + readStrict(s, readVarUInt(s)) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + serializeBlockInfo(ba) + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readUInt8(s) # has_nested + return code, text + + +def readHeader(s): + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + code, text = readException(s) + raise RuntimeError("Unexpected exception {}: {}".format(code, text)) + assert packet_type == 1, "Expected Data (header) packet, got {}".format(packet_type) + readStringBinary(s) # external table name + readBlockInfo(s) + columns = readVarUInt(s) + rows = readVarUInt(s) + assert rows == 0, "Expected an empty header block, got {} rows".format(rows) + # Consume the per-column descriptor so the stream stays aligned. + for _ in range(columns): + readStringBinary(s) # column name + readStringBinary(s) # column type + if readUInt8(s): # has custom serialization (>= 54454) + readKindStack(s) + + +def readExceptionPacket(s, expected_code): + packet_type = readVarUInt(s) + assert packet_type == 2, "Expected Exception packet, got {}".format(packet_type) + code, text = readException(s) + assert code == expected_code, "Expected code {}, got {}: {}".format(expected_code, code, text) + return text + + +def connectAndHandshake(s): + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + sendAddendum(s) + + +def sendPacketTypeOnly(s, packet_type, trailing=b""): + ba = bytearray() + writeVarUInt(packet_type, ba) + ba.extend(trailing) + s.sendall(ba) + + +def assertConnectionClosed(s): + # `socket.timeout` is not caught on purpose: it is how a server that keeps waiting for the + # payload of the rejected packet fails this test. + assert s.recv(1) == b"", "Expected the server to close the connection" + + +def sendUInt64Block(s, values): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) # external table name + serializeBlockInfo(ba) + writeVarUInt(1, ba) # columns + writeVarUInt(len(values), ba) # rows + writeStringBinary("x", ba) + writeStringBinary("UInt64", ba) + ba.append(0) # no custom serialization + for value in values: + ba.extend(struct.pack(" Date: Thu, 15 Oct 2026 18:28:45 +0000 Subject: [PATCH 23/81] Backport #122128 to 26.8: materializeBlock: instantiate type-only header columns before materializing --- src/Core/Block.cpp | 11 +++++-- ...eplicas_right_join_null_conjunct.reference | 3 ++ ...llel_replicas_right_join_null_conjunct.sql | 30 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.reference create mode 100644 tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.sql diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index ae8300c9862d4..78ca84e6bb165 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -1170,6 +1170,9 @@ Block materializeBlock(const Block & block, bool remove_special_column_represent for (size_t i = 0; i < columns; ++i) { auto & element = res.getByPosition(i); + /// Plan-derived headers may declare a column by type only; instantiate it before use. + if (!element.column) + element.column = element.type->createColumn(); element.column = element.column->convertToFullColumnIfConst(); if (remove_special_column_representations) element.column = removeSpecialRepresentations(element.column); @@ -1182,9 +1185,13 @@ void materializeBlockInplace(Block & block, bool remove_special_column_represent { for (size_t i = 0; i < block.columns(); ++i) { - block.getByPosition(i).column = block.getByPosition(i).column->convertToFullColumnIfConst(); + auto & element = block.getByPosition(i); + /// Plan-derived headers may declare a column by type only; instantiate it before use. + if (!element.column) + element.column = element.type->createColumn(); + element.column = element.column->convertToFullColumnIfConst(); if (remove_special_column_representations) - block.getByPosition(i).column = removeSpecialRepresentations(block.getByPosition(i).column); + element.column = removeSpecialRepresentations(element.column); } } diff --git a/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.reference b/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.reference new file mode 100644 index 0000000000000..200035ffa0a0d --- /dev/null +++ b/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.reference @@ -0,0 +1,3 @@ +2 +4 +2 diff --git a/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.sql b/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.sql new file mode 100644 index 0000000000000..7f197688a5b87 --- /dev/null +++ b/tests/queries/0_stateless/04768_parallel_replicas_right_join_null_conjunct.sql @@ -0,0 +1,30 @@ +-- RIGHT JOIN whose ON clause carries a constant NULL conjunct segfaulted while the plan-based +-- parallel-replicas join built its output header (#121906): the left header can declare the +-- residual column by type only, and materializeBlockInplace dereferenced the missing column. + +DROP TABLE IF EXISTS tb_l; +DROP TABLE IF EXISTS tb_r; + +CREATE TABLE tb_l (b String) ENGINE = MergeTree ORDER BY b; +CREATE TABLE tb_r (b String) ENGINE = MergeTree ORDER BY b; + +INSERT INTO tb_l VALUES ('a'), ('b'); +INSERT INTO tb_r VALUES ('a'), ('c'); + +SET enable_analyzer = 1; +SET automatic_parallel_replicas_mode = 0; +SET enable_parallel_replicas = 1; +SET parallel_replicas_for_non_replicated_merge_tree = 1; +SET max_parallel_replicas = 3; +SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +SET parallel_replicas_plan_based = 1; + +-- The conjunct never matches, so the RIGHT JOIN yields every right row; before the fix this died. +SELECT count() FROM tb_l AS l RIGHT JOIN tb_r AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)); +-- FULL JOIN walks the same left-materialization path and keeps unmatched rows from both sides. +SELECT count() FROM tb_l AS l FULL JOIN tb_r AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)); +-- Control: a matching RIGHT JOIN still joins correctly. +SELECT count() FROM tb_l AS l RIGHT JOIN tb_r AS r ON l.b = r.b; + +DROP TABLE tb_l; +DROP TABLE tb_r; From 6693782e452c49018ebb80725d2f2dd2da6a26f2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 20:14:51 +0000 Subject: [PATCH 24/81] Backport #126032 to 26.8: Carry the configured name on a cluster built from a host list --- src/Interpreters/Cluster.cpp | 7 +++ .../test.py | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 31dfa0636417a..584b8fe470c9b 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -586,6 +586,11 @@ Cluster::Cluster( UInt32 current_shard_num = 1; secret = params.cluster_secret; + /// `getName` is what a parallel replicas read ships to the shards in `cluster_for_parallel_replicas`, so a + /// cluster assembled from a host list has to carry the name it is configured under, the same way one read + /// from the config does. An ad-hoc cluster - the one `remote()` builds from a literal host list - is passed + /// no name, and stays unnamed because it has no name a shard could resolve. + name = params.cluster_name; for (const auto & shard : names) { @@ -618,6 +623,8 @@ Cluster::Cluster( UInt32 current_shard_num = 1; secret = params.cluster_secret; + /// Carries the configured name for the same reason as the constructor above. + name = params.cluster_name; for (const auto & shard : infos) { diff --git a/tests/integration/test_parallel_replicas_over_distributed/test.py b/tests/integration/test_parallel_replicas_over_distributed/test.py index 23aa602ee6894..88504619ee78d 100644 --- a/tests/integration/test_parallel_replicas_over_distributed/test.py +++ b/tests/integration/test_parallel_replicas_over_distributed/test.py @@ -156,3 +156,58 @@ def test_parallel_replicas_over_distributed( ) == expected_result ) + + +def test_parallel_replicas_over_distributed_on_replicated_database_cluster( + start_cluster, +): + # A `Replicated` database publishes its cluster under the database name, but assembles it from the + # replica list in Keeper instead of reading it from `remote_servers`. Such a cluster carried no + # name, and a distributed read ships `Cluster::getName` to the shards in + # `cluster_for_parallel_replicas`, so the shard received an empty setting and refused to plan the + # read at all: "Reading in parallel from replicas is enabled but cluster to execute query is not + # provided" (CLUSTER_DOESNT_EXIST). + database_name = "test_replicated_database_cluster" + table_name = "test_replicated_database_cluster_table" + group = nodes[0:3] + + for node in nodes: + node.query(f"DROP TABLE IF EXISTS {table_name}_d SYNC") + node.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + + for i, node in enumerate(group): + node.query( + f"CREATE DATABASE {database_name} " + f"ENGINE = Replicated('/{database_name}/db', 'shard1', 'r{i}')" + ) + + group[0].query( + f"CREATE TABLE {database_name}.{table_name} (key Int64) " + f"ENGINE = ReplicatedMergeTree ORDER BY key" + ) + group[0].query( + f"INSERT INTO {database_name}.{table_name} SELECT number FROM numbers(1000)" + ) + for node in group: + node.query(f"SYSTEM SYNC REPLICA {database_name}.{table_name}") + + # The distributed table is kept outside the replicated database, so only the initiator has it. Its + # cluster is the one the database publishes under its own name. + nodes[0].query( + f"CREATE TABLE {table_name}_d (key Int64) " + f"Engine=Distributed({database_name}, {database_name}, {table_name}, key)" + ) + + # prefer_localhost_replica = 0 is mandatory for the same reason as in the test above: the shard has + # to be read over the wire for the settings shipped with it to matter. + settings = { + "enable_parallel_replicas": 2, + "max_parallel_replicas": 3, + "prefer_localhost_replica": 0, + } + assert ( + nodes[0].query( + f"SELECT count(), sum(key) FROM {table_name}_d", settings=settings + ) + == "1000\t499500\n" + ) \ No newline at end of file From 4894962bc555a726110feeac8e0ec23ff500d7dc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 07:53:23 +0000 Subject: [PATCH 25/81] Backport #125221 to 26.8: Fix `ORDER BY ... LIMIT` over an `Interval` column with top-K dynamic filtering --- src/Functions/FunctionTopKFilter.cpp | 10 +- .../QueryPlan/Optimizations/optimizeTopK.cpp | 2 +- src/Processors/TopKThresholdTracker.h | 7 +- ...61_top_k_dynamic_filter_interval.reference | 123 ++++++++++++++++++ .../05361_top_k_dynamic_filter_interval.sql | 107 +++++++++++++++ 5 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference create mode 100644 tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql diff --git a/src/Functions/FunctionTopKFilter.cpp b/src/Functions/FunctionTopKFilter.cpp index bdf343beb8a3a..4c9d547f18cae 100644 --- a/src/Functions/FunctionTopKFilter.cpp +++ b/src/Functions/FunctionTopKFilter.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,7 @@ class FunctionTopKFilter final : public IFunction if (!threshold_tracker_) throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionTopKFilter got NULL threshold_tracker"); + threshold_type = recursiveRemoveLowCardinality(threshold_tracker_->getDataType()); direction = threshold_tracker_->getDirection(); nulls_direction = threshold_tracker_->getNullsDirection(); collator = threshold_tracker_->getCollator(); @@ -129,7 +131,9 @@ class FunctionTopKFilter final : public IFunction const DataTypePtr & data_type, size_t input_rows_count) const { - ColumnPtr threshold_column = data_type->createColumnConst(input_rows_count, convertFieldToType(current_threshold, *data_type)); + /// The threshold is a value of the tracker's type, which a `Field` does not carry (an `Interval` is a plain `Int64`). + ColumnPtr threshold_column = data_type->createColumnConst( + input_rows_count, convertFieldToType(current_threshold, *data_type, threshold_type.get())); ColumnsWithTypeAndName args{argument, {threshold_column, data_type, {}}}; auto elem_compare = compare_function->build(args); return elem_compare->execute(args, elem_compare->getResultType(), input_rows_count, false); @@ -144,7 +148,7 @@ class FunctionTopKFilter final : public IFunction { const auto & col = *argument.column; - auto threshold_field = convertFieldToType(current_threshold, *data_type); + auto threshold_field = convertFieldToType(current_threshold, *data_type, threshold_type.get()); auto threshold_col = data_type->createColumn(); threshold_col->insert(threshold_field); @@ -173,6 +177,8 @@ class FunctionTopKFilter final : public IFunction } TopKThresholdTrackerPtr threshold_tracker; + /// The type of the threshold value as this function receives it (the default implementation removes LowCardinality). + DataTypePtr threshold_type; FunctionOverloadResolverPtr compare_function; int direction; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp index b229f343cb2f4..30f4af59d4f02 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp @@ -208,7 +208,7 @@ size_t tryOptimizeTopK(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, /// Initial top-k mark selection (getTopKMarks) does not require it. if ((use_skip_index && settings.use_skip_indexes_on_data_read) || use_dynamic_filtering) { - threshold_tracker = std::make_shared(sort_col_desc); + threshold_tracker = std::make_shared(sort_col_desc, sort_column.type); sorting_step->setTopKThresholdTracker(threshold_tracker); } diff --git a/src/Processors/TopKThresholdTracker.h b/src/Processors/TopKThresholdTracker.h index 440c66a98c2fa..f295003177ef6 100644 --- a/src/Processors/TopKThresholdTracker.h +++ b/src/Processors/TopKThresholdTracker.h @@ -9,8 +9,8 @@ namespace DB struct TopKThresholdTracker { - explicit TopKThresholdTracker(const SortColumnDescription & sort_desc_) - : sort_desc(sort_desc_) + TopKThresholdTracker(const SortColumnDescription & sort_desc_, DataTypePtr data_type_) + : sort_desc(sort_desc_), data_type(std::move(data_type_)) { } @@ -57,6 +57,8 @@ struct TopKThresholdTracker int getDirection() const { return sort_desc.direction; } int getNullsDirection() const { return sort_desc.nulls_direction; } const std::shared_ptr & getCollator() const { return sort_desc.collator; } + /// The type of the sort column the published values are taken from. + const DataTypePtr & getDataType() const { return data_type; } private: /// Compare two Field values respecting NULL ordering and collation @@ -67,6 +69,7 @@ struct TopKThresholdTracker mutable SharedMutex mutex; std::atomic is_set{false}; SortColumnDescription sort_desc; + DataTypePtr data_type; }; using TopKThresholdTrackerPtr = std::shared_ptr; diff --git a/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference new file mode 100644 index 0000000000000..2f9c0b148d639 --- /dev/null +++ b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference @@ -0,0 +1,123 @@ +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +\N +\N +0 +0 +1 +2 +\N +\N +99999 +99999 +99998 +99997 +\N +\N +0 +0 +1 +2 +\N +\N +99999 +99999 +99998 +99997 +(0,0) +(0,1) +(0,2) +(1666,39) +(1666,38) +(1666,37) +[99999] +[99998] +[99997] +99999 +99998 +99997 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +\N +\N +0 +99999 +99998 +99997 diff --git a/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql new file mode 100644 index 0000000000000..57ffcb4d251b5 --- /dev/null +++ b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql @@ -0,0 +1,107 @@ +-- Random settings limits: use_top_k_dynamic_filtering=(1, 1); query_plan_max_limit_for_top_k_optimization=(1000, None); max_block_size=(None, 8192) + +-- `ORDER BY ... LIMIT` over a column whose type is or contains an `Interval` type, with the top-K dynamic filter. + +DROP TABLE IF EXISTS t_interval; +DROP TABLE IF EXISTS t_interval_key; +DROP TABLE IF EXISTS t_interval_minmax; +DROP TABLE IF EXISTS t_interval_memory; +DROP TABLE IF EXISTS t_interval_lc; + +CREATE TABLE t_interval +( + id UInt64, + d_ns IntervalNanosecond, + d_us IntervalMicrosecond, + d_ms IntervalMillisecond, + d_s IntervalSecond, + d_min IntervalMinute, + d_h IntervalHour, + d_d IntervalDay, + d_w IntervalWeek, + d_mon IntervalMonth, + d_q IntervalQuarter, + d_y IntervalYear, + n_s Nullable(IntervalSecond), + n_ns Nullable(IntervalNanosecond), + tup Tuple(IntervalHour, IntervalMinute), + arr Array(IntervalSecond) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_interval SELECT + number, + toIntervalNanosecond(number), toIntervalMicrosecond(number), toIntervalMillisecond(number), toIntervalSecond(number), + toIntervalMinute(number), toIntervalHour(number), toIntervalDay(number), toIntervalWeek(number), + toIntervalMonth(number), toIntervalQuarter(number), toIntervalYear(number), + if(number % 50000 = 7, NULL, toIntervalSecond(number)), + if(number % 50000 = 7, NULL, toIntervalNanosecond(number)), + (toIntervalHour(intDiv(number, 60)), toIntervalMinute(number % 60)), + [toIntervalSecond(number)] +FROM numbers(100000); + +SELECT d_ns FROM t_interval ORDER BY d_ns LIMIT 3; +SELECT d_ns FROM t_interval ORDER BY d_ns DESC LIMIT 3; +SELECT d_us FROM t_interval ORDER BY d_us LIMIT 3; +SELECT d_us FROM t_interval ORDER BY d_us DESC LIMIT 3; +SELECT d_ms FROM t_interval ORDER BY d_ms LIMIT 3; +SELECT d_ms FROM t_interval ORDER BY d_ms DESC LIMIT 3; +SELECT d_s FROM t_interval ORDER BY d_s LIMIT 3; +SELECT d_s FROM t_interval ORDER BY d_s DESC LIMIT 3; +SELECT d_min FROM t_interval ORDER BY d_min LIMIT 3; +SELECT d_min FROM t_interval ORDER BY d_min DESC LIMIT 3; +SELECT d_h FROM t_interval ORDER BY d_h LIMIT 3; +SELECT d_h FROM t_interval ORDER BY d_h DESC LIMIT 3; +SELECT d_d FROM t_interval ORDER BY d_d LIMIT 3; +SELECT d_d FROM t_interval ORDER BY d_d DESC LIMIT 3; +SELECT d_w FROM t_interval ORDER BY d_w LIMIT 3; +SELECT d_w FROM t_interval ORDER BY d_w DESC LIMIT 3; +SELECT d_mon FROM t_interval ORDER BY d_mon LIMIT 3; +SELECT d_mon FROM t_interval ORDER BY d_mon DESC LIMIT 3; +SELECT d_q FROM t_interval ORDER BY d_q LIMIT 3; +SELECT d_q FROM t_interval ORDER BY d_q DESC LIMIT 3; +SELECT d_y FROM t_interval ORDER BY d_y LIMIT 3; +SELECT d_y FROM t_interval ORDER BY d_y DESC LIMIT 3; + +SELECT n_s FROM t_interval ORDER BY n_s ASC NULLS FIRST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s ASC NULLS LAST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s DESC NULLS FIRST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s DESC NULLS LAST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns ASC NULLS FIRST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns ASC NULLS LAST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns DESC NULLS FIRST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns DESC NULLS LAST LIMIT 3; + +SELECT tup FROM t_interval ORDER BY tup LIMIT 3; +SELECT tup FROM t_interval ORDER BY tup DESC LIMIT 3; + +SELECT arr FROM t_interval ORDER BY arr DESC LIMIT 3 SETTINGS use_top_k_dynamic_filtering_for_variable_length_types = 1; + +CREATE TABLE t_interval_key (d IntervalSecond) ENGINE = MergeTree ORDER BY d; +INSERT INTO t_interval_key SELECT toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_key ORDER BY d DESC LIMIT 3 SETTINGS optimize_read_in_order = 0; + +CREATE TABLE t_interval_minmax (id UInt64, d IntervalSecond, INDEX idx d TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_interval_minmax SELECT number, toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_minmax ORDER BY d DESC LIMIT 3 SETTINGS use_skip_indexes_for_top_k = 1, use_skip_indexes_on_data_read = 1; + +CREATE TABLE t_interval_memory (d IntervalSecond) ENGINE = Memory; +INSERT INTO t_interval_memory SELECT toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_memory ORDER BY d LIMIT 3; +SELECT d FROM t_interval_memory ORDER BY d DESC LIMIT 3; + +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE t_interval_lc (id UInt64, d LowCardinality(IntervalSecond), n LowCardinality(Nullable(IntervalSecond))) +ENGINE = MergeTree ORDER BY id; +INSERT INTO t_interval_lc SELECT number, toIntervalSecond(number), if(number % 50000 = 7, NULL, toIntervalSecond(number)) +FROM numbers(100000); +SELECT d FROM t_interval_lc ORDER BY d LIMIT 3; +SELECT d FROM t_interval_lc ORDER BY d DESC LIMIT 3; +SELECT n FROM t_interval_lc ORDER BY n ASC NULLS FIRST LIMIT 3; +SELECT n FROM t_interval_lc ORDER BY n DESC NULLS LAST LIMIT 3; + +DROP TABLE t_interval; +DROP TABLE t_interval_key; +DROP TABLE t_interval_minmax; +DROP TABLE t_interval_memory; +DROP TABLE t_interval_lc; From e24fd1df1adde87b8031c66373a48d2a64843b21 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 11:55:35 +0000 Subject: [PATCH 26/81] Backport #121704 to 26.8: Stop the native SSH-key handshake from disclosing whether a user exists --- src/Server/TCPHandler.cpp | 13 -- .../0_stateless/03780_failed_ssh.expect | 61 ++++-- .../04908_ssh_key_auth_user_enumeration.py | 186 ++++++++++++++++++ ...08_ssh_key_auth_user_enumeration.reference | 2 + 4 files changed, 235 insertions(+), 27 deletions(-) create mode 100755 tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py create mode 100644 tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 27ce8ff4daaac..2ce8857f4bc22 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2258,19 +2258,6 @@ void TCPHandler::receiveHello() /// Perform handshake for SSH authentication if (is_ssh_based_auth) { - const auto authentication_types = session->getAuthenticationTypesOrLogInFailure(user); - - bool user_supports_ssh_authentication = std::find_if( - authentication_types.begin(), - authentication_types.end(), - [](auto authentication_type) - { - return authentication_type == AuthenticationType::SSH_KEY; - }) != authentication_types.end(); - - if (!user_supports_ssh_authentication) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Expected authentication with SSH key"); - if (client_tcp_protocol_version < DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Cannot authenticate user with SSH key, because client version is too old"); diff --git a/tests/queries/0_stateless/03780_failed_ssh.expect b/tests/queries/0_stateless/03780_failed_ssh.expect index e1b638438fa22..c4bb68e7851ae 100755 --- a/tests/queries/0_stateless/03780_failed_ssh.expect +++ b/tests/queries/0_stateless/03780_failed_ssh.expect @@ -23,20 +23,53 @@ expect_after { } spawn bash -c "echo '-----BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcn -NhAAAAAwEAAQAAAIEAxOYq7137PbwKYw6IahB/pHvYNvFTrMg6bAT182PLg+Fidb+wbMNw -qkeL4C8wIxwulaScbjpidEEVbB3egOXf92cVmO7FxnwJkbuOkQdTZ68+vh9oy7jRwLsTq0 -K5Ez0nnauHyoR2Qdd+7AnTSSyEi1Efdni5Ble/LhHhRPU//ckAAAIQRkn3H0ZJ9x8AAAAH -c3NoLXJzYQAAAIEAxOYq7137PbwKYw6IahB/pHvYNvFTrMg6bAT182PLg+Fidb+wbMNwqk -eL4C8wIxwulaScbjpidEEVbB3egOXf92cVmO7FxnwJkbuOkQdTZ68+vh9oy7jRwLsTq0K5 -Ez0nnauHyoR2Qdd+7AnTSSyEi1Efdni5Ble/LhHhRPU//ckAAAADAQABAAAAgDW3MSliM0 -NxREgn09FTtO7TlnXOumwdp8qGQ+7lX8UXvLuw0tmpK9hYcnFzjidV7eOHJ+HubaOXideY -AnaZv9KNNWBDTOeuHpMtaIzjq3yb/BuOSC1Xthjb7UZoQGlgAUOdCTFap3jxV8PeLbFazg -oMqjTVCwiWxlIumBoZgwTNAAAAQQCxyh6w3GOFtqts66TVNs8pSe8r/cSOUnTU1cws5s0t -l+CHod8qOHF+eWWnw+TvPw3sTM/5rcJ+0N3anhpo9Ph3AAAAQQDl0EKJvfyRgnKHcjtLUM -IPac9E4ZR36UA2lWK/gr0x8PvxzlonzWqBcYFmCAQv9lPvk1GJK8/EnCkkD5pTvKbHAAAA -QQDbVchcKjXrhsYklDvJQ4xyGkUH6+xMRVINwzjP3JQ1akUF/MJAYg5vr8j4gibWysVwWV -0lgWuG96Yy/MxT62bvAAAAE2lzYWtlbEBwYzY0MTAxLTI1MzYBAgMEBQYH +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAoUTN2fewg1IpJFd006wAQ7O2Y9ntq5KUQ2Tpvmxd/hrrPY3PdQmt +Vvnws1Sn3pn1NWSKxYD+Yfh5ZRhioWv9lJe2IxgbvA1tEIgX0Sx5T5fbkCZMaOzMXFsEvp +KWPlBTkz557lvQQVOvzAHnH20LC4o84/QileP4rhQR86CtAoJkweLPv/+0nMTeEC9FPGwm +ls2u2ezz2mLfHKU1+vlrXa+qzfvyl8aWDdeR5VDfRQ6foXSoji6z1zlf1JSulrUq76DdMJ +1iH+Y8sEYbNpvVaW6nSqpIhrgg6Z1qY+sBIheTSyiX1IS9DeDrXMxi+NKGEmiGM+Rn+RKR +0V7fk8fJLlUSf5g3KDTjd7DxvS7Zu73kNR/ODUWpUWNw+/OlhGCseNQZgOhXdkF9m87wH0 +9wB+Ny5gH06z8A5jdi1k7h3+pD1Q/CV2tL80xrEga0wIeaytAigiqqc6FiyIKtDmUIr+2v +NR/iHW59tqPehG5kFV5+zuqVe8xHehV40tQza4LQaXDvR6US1u5Bk3zRE8p0ZHGF1VnRXR +vSgvMEJFYzEPIj99sCWUw3CD2NdmKRGI3kpDrCyMHbCwWR1HYhtazpMiauAxFxVsNHInCA +2HTar629DKpgzW3C3yQBPhRofg2kZlbQCS1MxZzaOlRpBKDnIkqicwlpUEARnL6QFnNlhD +8AAAc4P+ey+j/nsvoAAAAHc3NoLXJzYQAAAgEAoUTN2fewg1IpJFd006wAQ7O2Y9ntq5KU +Q2Tpvmxd/hrrPY3PdQmtVvnws1Sn3pn1NWSKxYD+Yfh5ZRhioWv9lJe2IxgbvA1tEIgX0S +x5T5fbkCZMaOzMXFsEvpKWPlBTkz557lvQQVOvzAHnH20LC4o84/QileP4rhQR86CtAoJk +weLPv/+0nMTeEC9FPGwmls2u2ezz2mLfHKU1+vlrXa+qzfvyl8aWDdeR5VDfRQ6foXSoji +6z1zlf1JSulrUq76DdMJ1iH+Y8sEYbNpvVaW6nSqpIhrgg6Z1qY+sBIheTSyiX1IS9DeDr +XMxi+NKGEmiGM+Rn+RKR0V7fk8fJLlUSf5g3KDTjd7DxvS7Zu73kNR/ODUWpUWNw+/OlhG +CseNQZgOhXdkF9m87wH09wB+Ny5gH06z8A5jdi1k7h3+pD1Q/CV2tL80xrEga0wIeaytAi +giqqc6FiyIKtDmUIr+2vNR/iHW59tqPehG5kFV5+zuqVe8xHehV40tQza4LQaXDvR6US1u +5Bk3zRE8p0ZHGF1VnRXRvSgvMEJFYzEPIj99sCWUw3CD2NdmKRGI3kpDrCyMHbCwWR1HYh +tazpMiauAxFxVsNHInCA2HTar629DKpgzW3C3yQBPhRofg2kZlbQCS1MxZzaOlRpBKDnIk +qicwlpUEARnL6QFnNlhD8AAAADAQABAAACAEaNMj1cReV9iTAz7pE32znDIlTWn0qDKse9 +GyFKiWhEYIuiQZMjM6ksWSgrLfGNix/bp3Y2M5Oqv3kBzQxvkRVztaPuxRtlCETGNR02hq +/ScKTEjIMOrWzuCMpSmNYUECwHlhI4KoiZ0vu+DV3LN5zJ0k+rm4HVpRKLBzt9HJNPdkL4 +Ngj6FgrLeZa9B14itlMdEpSWPCxbOt5uTIpKmbT3yBKrWXsazsSpfVNEKRkuB5n7zk29ks +HigNEgZqzZ/zmsN3gwMiBgekZntToTvK6OUTOyAV6mqQdq7YNe6TT/+VlVHe9ETu0GbBFB +/T5741mkTqnfD7mAbTiad9dP/OvND1c40v1n4e3ZHU/FtUwCArr2oKP0Q6+Y1ool5Szi/M +MUsjepnKqFuaLpkuDThaPLBissz+nP3HRlwqeaxFKTk/Hl4IraGK1BDVc1x4mQsFaobL/3 ++hxZf/fN+lDTHjr93OXqgU/yUK9jmp7n3KICZNZz6wyqD/bVL1WIGPxHdCnq77S7TYTH48 +XGErc0P9xsnXVlqE4Vxuw2bP6FPrQ8EzjGj53H5k0kBa3fospC6rwEUAB4g0W1aW5ZzU0D +EASqTnFEX33ZB06iBDLZqDEEl7VSKVDR3GBHBLJXAvQojmFHpRXAm3SXfugYTLfF+qmgnu +iNfYRQz+lMzxD8331hAAABABosTEeM32bFSQOIK5uzJUphbhhJvwpLv+6jzr+7GLYMy26q +QfFfmRK7qHHfTqHSngQ5h0H5AOrkLR/xw6vetDzKEuRnnDgsAPaT0wt8wKV66vSrmO6GFX +cYi/qg2D8YJjeMSyKGD8Rd6Nq+ukssQ9LpNCkbYL6Zom/Qvd2ZlQGkoh6iVdOX78ULbdN1 +XmJwjrhggIKJWSFrv4cjZEP4Q3g11FC+zs+R71/i0YDcnDiNbh+1khNfMDxpzaaV1L3bwE +ouyYlk17eQghM7rgpnh76/1dlI+/O+S9P6o7fQT1aIiG5ms4LOMDtFyV88nq2Q+3RXYhTT +ZGvlaC4L5weIcecAAAEBANbOknEph4Hc9kuWGOcH6TmbbJrCktE/jv66YlwbegS9q8krJ7 +6thAS6gNi3vojVpi2i4w8eNkttjEwNsg7WMVjzV9XTF8R6HbPn33s+EHaLiL1zn7wcmAIB +HNalc5zjC5pfOf6vfe4JyZNRUyoHUQUNzqJRuea8h7lmup07STCUWQOypmEZK6zsi7We6Y +pkl+byMBXFUifE5pW749uygtT3vR7djQiYGPevYIVKqw3LQ5R7QeZ4ZZJaTXbrD2dHWyr9 +DroHgfGMsfVHvLTUc+TmL+SrBNUOHftMxo4Szy2b2IPP5HWedDwD1ft5x2O98xU6jIJmnm +F5tk/sAv8CgG8AAAEBAMAx58ag2qGMcOv36DhhhrwMKlLQgb3Wqooo6FvR6TaxjqAu/8b4 +UFgi/KV4Irdrd22hvYJDNseJNKqCKGOwvPiHE3uu1ERmmrG/xw7NAlXZPDoHclK/tr/xPE +IEGOqRQHbHaCE/0N91ncF35Hu3dXQau0p+DszUt9RZpO4YllJEMe+Xy/sxlv3ul6H522ZL +a4ifUw559aqIXXvsn8iFT1iLt1t3nDjepptWHggOb4VbuEAY24HSbxW9OwTcEVEvlpXNg0 +kcintN4pJb/F3J1SGPIy/Hj6fLTHkMGcYqJz2oKMRTv/wOdBFCQay5juDeOJTXhyCtDPoV +V0u3f4V9gTEAAAAAAQID -----END OPENSSH PRIVATE KEY-----' > id_rsa_03780_\$CLICKHOUSE_DATABASE" spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CURL -sS \$CLICKHOUSE_URL -d \"CREATE USER IF NOT EXISTS user_03780_\$CLICKHOUSE_DATABASE\"" diff --git a/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py new file mode 100755 index 0000000000000..67dd8d3dd5b06 --- /dev/null +++ b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# Tags: no-fasttest, no-openssl-fips +# no-fasttest: the SSH handshake is only compiled into a server built with SSH support. +# no-openssl-fips: creates an ssh-ed25519 credential, which FIPS builds reject. +"""Test that SSH-key authentication does not reveal whether a user exists. + +A client asks for it by putting a marker in front of the user name, and the server answers with a +challenge to sign. Every name must get that far - one with an SSH key, one with a password, one +that nobody has - and all three must then fail identically apart from the name itself. + +The client asks for SSH-key authentication only when it holds a private key, so it cannot reach +the last two cases. This test speaks the native protocol over a raw socket instead, asking for a +challenge for each name and answering with garbage. +""" + +import os +import socket +import subprocess + +# `clickhouse-test` exports CLICKHOUSE_BINARY and CLICKHOUSE_DATABASE, but not the host and port - +# those come from the environment the suite was started in, and `shell_config.sh`, which supplies +# them, is only sourced by the `.sh` tests. Hence the defaults, as in the other `.py` tests here. +HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", 9000)) +DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT = [ + os.environ.get("CLICKHOUSE_BINARY", "clickhouse"), + "client", + "--host", + HOST, + "--port", + str(PORT), + "--database", + DATABASE, +] + +TIMEOUT = 20 + +# Packet types of the native protocol. +CLIENT_HELLO = 0 +CLIENT_SSH_CHALLENGE_REQUEST = 11 +CLIENT_SSH_CHALLENGE_RESPONSE = 12 +SERVER_EXCEPTION = 2 +SERVER_SSH_CHALLENGE = 18 + +# A user name behind this marker is what asks for SSH-key authentication. +SSH_KEY_MARKER = " SSH KEY AUTHENTICATION " + +# The oldest protocol revision that supports the SSH handshake; the server refuses older ones. +REVISION = 54466 + +AUTHENTICATION_FAILED = 516 +GENERIC_ERROR = ( + "Authentication failed: password is incorrect, or there is no user with such name" +) + +# The three user states that must be indistinguishable. The test database is unique per run, so +# naming the users after it keeps parallel runs apart. The password user is not `default`, whose +# error message carries extra password-reset help. +SSH_USER = f"ssh_user_{DATABASE}" +PASSWORD_USER = f"password_user_{DATABASE}" +UNKNOWN_USER = f"nosuchuser_{DATABASE}" + +# Any valid public key will do, because no signature the test sends can match it. +PUBLIC_KEY_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "../../config/ssh_user_ed25519_key.pub" +) + + +def run_query(query): + subprocess.run(CLIENT + ["--query", query], check=True) + + +def write_varuint(value): + encoded = bytearray() + while value >= 0x80: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def write_string(value): + data = value.encode() + return write_varuint(len(data)) + data + + +def read_exact(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError(f"connection closed after {len(data)} of {size} bytes") + data += chunk + return data + + +def read_varuint(sock): + value = 0 + for i in range(9): + byte = read_exact(sock, 1)[0] + value |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + break + return value + + +def read_string(sock): + return read_exact(sock, read_varuint(sock)).decode() + + +def read_exception(sock): + # Code, name and message. The stack trace and the nested flag that follow are not read, + # because the connection is closed right after. + code = int.from_bytes(read_exact(sock, 4), "little", signed=True) + read_string(sock) + message = read_string(sock) + return code, message + + +def ssh_handshake(user): + """Request SSH-key authentication as `user` and answer the challenge with a bad signature. + + Returns whether the server asked for a signature at all, and the error it ended with. + """ + hello = ( + write_varuint(CLIENT_HELLO) + + write_string("probe") # client name + + write_varuint(24) # client version major + + write_varuint(3) # client version minor + + write_varuint(REVISION) + + write_string("") # default database + + write_string(SSH_KEY_MARKER + user) + + write_string("") # empty password + ) + + with socket.create_connection((HOST, PORT), timeout=TIMEOUT) as sock: + sock.sendall(hello + write_varuint(CLIENT_SSH_CHALLENGE_REQUEST)) # and ask for a challenge + + packet = read_varuint(sock) + if packet == SERVER_EXCEPTION: + code, message = read_exception(sock) + return False, code, message + assert packet == SERVER_SSH_CHALLENGE, f"unexpected packet {packet}" + + read_exact(sock, read_varuint(sock)) # the challenge, which the test cannot sign + sock.sendall( + write_varuint(CLIENT_SSH_CHALLENGE_RESPONSE) + write_string("not-a-signature") + ) + + packet = read_varuint(sock) + assert packet == SERVER_EXCEPTION, f"unexpected packet {packet}" + code, message = read_exception(sock) + return True, code, message + + +def main(): + with open(PUBLIC_KEY_FILE, encoding="utf-8") as key_file: + public_key = key_file.read().split()[1] + + run_query(f"DROP USER IF EXISTS {SSH_USER}, {PASSWORD_USER}") + try: + run_query( + f"CREATE USER {SSH_USER} IDENTIFIED WITH ssh_key BY KEY '{public_key}' TYPE 'ssh-ed25519'" + ) + run_query(f"CREATE USER {PASSWORD_USER} IDENTIFIED WITH sha256_password BY 'password'") + + errors = set() + for user in (SSH_USER, PASSWORD_USER, UNKNOWN_USER): + challenged, code, message = ssh_handshake(user) + # Being refused before the challenge is itself a disclosure, whatever the error says. + assert challenged, f"{user} was refused before the challenge: {message}" + assert code == AUTHENTICATION_FAILED, f"{user}: error {code}: {message}" + assert GENERIC_ERROR in message, f"{user}: unexpected error: {message}" + # The name is the one thing the caller already knows, so it may differ - nothing else. + errors.add(message.replace(user, "USER")) + finally: + run_query(f"DROP USER IF EXISTS {SSH_USER}, {PASSWORD_USER}") + + print("every user was challenged and then got the generic authentication error") + assert len(errors) == 1, f"the errors differ: {errors}" + print("the errors are identical apart from the user name") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference new file mode 100644 index 0000000000000..9106017f70ba1 --- /dev/null +++ b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference @@ -0,0 +1,2 @@ +every user was challenged and then got the generic authentication error +the errors are identical apart from the user name From 12309323ddc6f49e19eb158dc6b765ede9f9d32a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 16:17:09 +0000 Subject: [PATCH 27/81] Backport #125785 to 26.8: Use constant-time comparison for TOTP codes and plaintext passwords --- src/Access/Authentication.cpp | 3 ++- src/Access/Common/OneTimePassword.cpp | 13 +++++++++---- src/Common/StringUtils.h | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index a0bcd3479becc..fa33d8eca45bf 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -17,6 +17,7 @@ #include #include +#include #if USE_SSL # include @@ -205,7 +206,7 @@ namespace case AuthenticationType::PLAINTEXT_PASSWORD: { const auto & password_plaintext = authentication_method.getPasswordHashBinary(); - return Util::stringToDigest(password) == password_plaintext ? on_success : Authentication::CredentialsCheckResult::Fail; + return constantTimeEquals(Util::stringToDigest(password), password_plaintext) ? on_success : Authentication::CredentialsCheckResult::Fail; } case AuthenticationType::SHA256_PASSWORD: { diff --git a/src/Access/Common/OneTimePassword.cpp b/src/Access/Common/OneTimePassword.cpp index 636fcb5de7161..8ba6630fdc862 100644 --- a/src/Access/Common/OneTimePassword.cpp +++ b/src/Access/Common/OneTimePassword.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -151,15 +152,19 @@ static String getOneTimePassword(const String & secret [[ maybe_unused ]], const static std::optional findMatchingTimeStep(std::string_view password, const OneTimePasswordSecret & secret, UInt64 current_time) { if (password.size() != static_cast(secret.params.num_digits) - || !std::all_of(password.begin(), password.end(), ::isdigit)) + || !std::all_of(password.begin(), password.end(), isNumericASCII)) return {}; + std::optional result; for (int delta : {0, -1, 1}) { - if (password == getOneTimePassword(secret.key, secret.params, current_time + delta * secret.params.period)) - return static_cast(current_time / secret.params.period) + delta; + auto candidate_pass = getOneTimePassword(secret.key, secret.params, current_time + delta * secret.params.period); + bool matched = constantTimeEquals(password, candidate_pass); + /// No early return to avoid leaking which window matched + if (matched && !result.has_value()) + result = static_cast(current_time / secret.params.period) + delta; } - return {}; + return result; } bool checkOneTimePassword(std::string_view password, const OneTimePasswordSecret & secret) diff --git a/src/Common/StringUtils.h b/src/Common/StringUtils.h index 5d32a28f0fb9c..5ebabf559facc 100644 --- a/src/Common/StringUtils.h +++ b/src/Common/StringUtils.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -278,6 +279,30 @@ inline bool equalsCaseInsensitive(std::string_view a, std::string_view b) && std::equal(a.begin(), a.end(), b.begin(), [](char x, char y) { return equalsCaseInsensitive(x, y); }); } +/// Equality for secret-derived byte sequences such as passwords and one-time codes. Unlike `memcmp` +/// and `operator==`, the runtime does not depend on the position of the first mismatch, so timing does +/// not reveal how many leading bytes were right. The lengths are treated as public, as in OpenSSL's +/// `CRYPTO_memcmp`. The `volatile` reads keep the compiler from turning the loop into an early-exit compare. +inline bool constantTimeEquals(std::span lhs, std::span rhs) +{ + if (lhs.size() != rhs.size()) + return false; + + const volatile uint8_t * a = lhs.data(); + const volatile uint8_t * b = rhs.data(); + uint8_t diff = 0; + for (size_t i = 0; i < lhs.size(); ++i) + diff |= a[i] ^ b[i]; + return diff == 0; +} + +inline bool constantTimeEquals(std::string_view lhs, std::string_view rhs) +{ + return constantTimeEquals( + std::span(reinterpret_cast(lhs.data()), lhs.size()), + std::span(reinterpret_cast(rhs.data()), rhs.size())); +} + /// ASCII-only case conversion. Keywords, identifiers and function names are ASCII, and applying /// the locale to them would be wrong, so these deliberately leave every other byte alone. From 5396bbd5095335fe6684e899285545dd55560136 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 16:38:31 +0000 Subject: [PATCH 28/81] Backport #116488 to 26.8: Fix reading dropped column with existing column in data file --- src/Formats/FormatFilterInfo.h | 4 + .../Formats/Impl/Parquet/SchemaConverter.cpp | 51 +++++++--- .../Formats/Impl/Parquet/SchemaConverter.h | 3 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 5 + .../DataLakes/Iceberg/SchemaProcessor.cpp | 12 ++- .../DataLakes/Iceberg/SchemaProcessor.h | 4 + .../test_dropped_column_in_data_file.py | 92 +++++++++++++++++++ ...opped_column_in_data_file_113324.reference | 5 + ...berg_dropped_column_in_data_file_113324.sh | 88 ++++++++++++++++++ 9 files changed, 247 insertions(+), 17 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_dropped_column_in_data_file.py create mode 100644 tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.reference create mode 100755 tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.sh diff --git a/src/Formats/FormatFilterInfo.h b/src/Formats/FormatFilterInfo.h index 2126610d3696c..920f6441c4c63 100644 --- a/src/Formats/FormatFilterInfo.h +++ b/src/Formats/FormatFilterInfo.h @@ -54,6 +54,9 @@ class ColumnMapper bool hasIcebergRequiredInfo() const { return has_iceberg_required_info; } bool isIcebergOptionalPath(const String & path) const { return iceberg_optional_paths.contains(path); } + void setLastAssignedFieldId(Int64 last_assigned_field_id_) { last_assigned_field_id = last_assigned_field_id_; } + std::optional getLastAssignedFieldId() const { return last_assigned_field_id; } + /// clickhouse_column_name -> format_column_name (just join the maps above by field_id). std::pair, std::unordered_map> makeMapping(const std::unordered_map & format_encoding) const; @@ -64,6 +67,7 @@ class ColumnMapper bool has_iceberg_string_info = false; std::unordered_set iceberg_optional_paths; bool has_iceberg_required_info = false; + std::optional last_assigned_field_id; }; using ColumnMapperPtr = std::shared_ptr; diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index 3723016eb09d8..bf5cd131f135e 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -147,8 +147,10 @@ NamesAndTypesList SchemaConverter::inferSchema() return res; } -std::string_view SchemaConverter::useColumnMapperIfNeeded(const parq::SchemaElement & element, const String & current_path) const +std::string_view SchemaConverter::useColumnMapperIfNeeded( + const parq::SchemaElement & element, const String & current_path, bool & out_not_in_schema) const { + out_not_in_schema = false; if (!column_mapper) return element.name; const auto & map = column_mapper->getFieldIdToClickHouseName(); @@ -162,19 +164,30 @@ std::string_view SchemaConverter::useColumnMapperIfNeeded(const parq::SchemaElem auto it = map.find(element.field_id); if (it == map.end()) { - /// Iceberg reserves field ids greater than 2147483447 (Integer.MAX_VALUE - 200) for metadata - /// columns, e.g. the v3 row-lineage fields _row_id (2147483540) and - /// _last_updated_sequence_number (2147483539). Spec-compliant Iceberg writers physically - /// write these into data files, but they are not part of the table schema. Per the Iceberg - /// spec (https://iceberg.apache.org/spec/#reserved-field-ids), readers must ignore - /// reserved-range field ids they don't recognize rather than failing. Such a column is - /// never requested, so returning its physical name lets the existing "unrequested column" - /// path skip it. - static constexpr Int64 iceberg_max_user_field_id = 2147483447; /// Integer.MAX_VALUE - 200; ids above this are reserved + /// Reserved field ids (https://iceberg.apache.org/spec/#reserved-field-ids) are not part of + /// the table schema, e.g. the v3 row-lineage fields `_row_id` (2147483540) and + /// `_last_updated_sequence_number` (2147483539) that spec-compliant writers materialize + /// into data files. Those are requested by their physical name, so they are matched by name. + static constexpr Int64 iceberg_max_user_field_id = 2147483447; /// Integer.MAX_VALUE - 200 if (element.field_id > iceberg_max_user_field_id) return element.name; - throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "Parquet file has column {} with field_id {} that is not in datalake metadata", element.name, element.field_id); + /// An id at or below `last-column-id` of the table metadata belongs to a column dropped + /// from the table: `DROP COLUMN` is metadata-only, so data files keep the column. Any other + /// id means the file does not belong to this table, or its schema was resolved to a wrong + /// one, which is reported rather than silently ignored. + const auto last_assigned_field_id = column_mapper->getLastAssignedFieldId(); + if (element.field_id < 1 || !last_assigned_field_id.has_value() || element.field_id > *last_assigned_field_id) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Parquet file has column {} with field_id {} that is not in datalake metadata, and the table cannot have " + "assigned that field id: the highest field id it ever assigned is {}", + element.name, + element.field_id, + last_assigned_field_id.has_value() ? std::to_string(*last_assigned_field_id) : String("unknown")); + + out_not_in_schema = true; + return element.name; } /// At top level (empty path), return the full mapped name. For nested @@ -221,9 +234,13 @@ void SchemaConverter::processSubtree(TraversalNode & node) if (node.schema_context == SchemaContext::None) { - node.appendNameComponent(node.element->name, useColumnMapperIfNeeded(*node.element, node.name)); + bool not_in_schema = false; + node.appendNameComponent(node.element->name, useColumnMapperIfNeeded(*node.element, node.name, not_in_schema)); - if (sample_block) + /// A column that is not in the data lake schema is never requested, and its physical name + /// may coincide with an unrelated column of the current schema (e.g. a dropped `x` and a + /// later re-added `x` have different field ids), so it must not be matched by name. + if (sample_block && !not_in_schema) { /// Doing this lookup on each schema element to support reading individual tuple elements. /// E.g.: @@ -770,11 +787,15 @@ void SchemaConverter::processSubtreeTuple(TraversalNode & node) std::vector element_names_in_file; for (size_t i = 0; i < size_t(node.element->num_children); ++i) { - const String & element_name = element_names_in_file.emplace_back(useColumnMapperIfNeeded(file_metadata.schema.at(schema_idx), node.name)); + bool not_in_schema = false; + const String & element_name = element_names_in_file.emplace_back( + useColumnMapperIfNeeded(file_metadata.schema.at(schema_idx), node.name, not_in_schema)); std::optional idx_in_output_tuple = i - skipped_unsupported_columns; if (lookup_by_name) { - idx_in_output_tuple = tuple_type_hint->tryGetPositionByName(element_name, options.format.parquet.case_insensitive_column_matching); + idx_in_output_tuple = std::nullopt; + if (!not_in_schema) + idx_in_output_tuple = tuple_type_hint->tryGetPositionByName(element_name, options.format.parquet.case_insensitive_column_matching); if (idx_in_output_tuple.has_value() && elements.at(idx_in_output_tuple.value()) != UINT64_MAX) throw Exception(ErrorCodes::DUPLICATE_COLUMN, "Parquet tuple {} has multiple elements with name `{}`", node.getNameForLogging(), element_name); diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h index 6d4efafcd4512..fb7deeca8fafb 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.h +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.h @@ -154,7 +154,8 @@ struct SchemaConverter /// For nested tuple elements, returns just the element name like `x`, not the whole path like `t.x`. /// For top-level columns (when current_path is empty), returns the full mapped name to support /// column names with dots (e.g. `integer.col` in Iceberg). - std::string_view useColumnMapperIfNeeded(const parq::SchemaElement & element, const String & current_path) const; + std::string_view useColumnMapperIfNeeded( + const parq::SchemaElement & element, const String & current_path, bool & out_not_in_schema) const; }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index b47dec44d1a06..67ecc542c68cf 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -327,6 +327,9 @@ Int32 IcebergMetadata::parseTableSchema( { const auto format_version = metadata_object->getValue(f_format_version); + if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) + schema_processor.updateLastColumnId(metadata_object->getValue(f_last_column_id)); + if (format_version == 2) { auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); @@ -376,6 +379,8 @@ static Poco::JSON::Object::Ptr traverseMetadataAndFindNecessarySnapshotObject( { if (!metadata_object->has(f_snapshots)) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "No snapshot set found in metadata for iceberg file"); + if (metadata_object->has(f_last_column_id) && !metadata_object->isNull(f_last_column_id)) + schema_processor->updateLastColumnId(metadata_object->getValue(f_last_column_id)); auto schemas = metadata_object->get(f_schemas).extract(); for (UInt32 j = 0; j < schemas->size(); ++j) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index eba2434a0b42e..5978dc1a41851 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -989,12 +989,22 @@ std::unordered_set IcebergSchemaProcessor::collectIcebergOptionalPaths(P return result; } +void IcebergSchemaProcessor::updateLastColumnId(Int32 last_column_id_) +{ + Int64 current = last_column_id.load(); + while (last_column_id_ > current && !last_column_id.compare_exchange_weak(current, last_column_id_)) + ; +} + ColumnMapperPtr IcebergSchemaProcessor::getColumnMapperById(Int32 id) const { auto schema = getIcebergTableSchemaById(id); if (!schema) return nullptr; - return createColumnMapper(schema); + auto column_mapper = createColumnMapper(schema); + if (Int64 known_last_column_id = last_column_id.load(); known_last_column_id >= 0) + column_mapper->setLastAssignedFieldId(known_last_column_id); + return column_mapper; } ColumnMapperPtr createColumnMapperFromFields(Poco::JSON::Array::Ptr fields) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index cb186791710e3..e2353b4a6e357 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -132,7 +132,11 @@ class IcebergSchemaProcessor ColumnMapperPtr getColumnMapperById(Int32 id) const; + void updateLastColumnId(Int32 last_column_id_); + private: + std::atomic last_column_id{-1}; + std::unordered_map iceberg_table_schemas_by_ids TSA_GUARDED_BY(mutex); std::unordered_map> clickhouse_table_schemas_by_ids TSA_GUARDED_BY(mutex); std::map, std::shared_ptr> transform_dags_by_ids TSA_GUARDED_BY(mutex); diff --git a/tests/integration/test_storage_iceberg_with_spark/test_dropped_column_in_data_file.py b/tests/integration/test_storage_iceberg_with_spark/test_dropped_column_in_data_file.py new file mode 100644 index 0000000000000..2caa3f82435da --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_dropped_column_in_data_file.py @@ -0,0 +1,92 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_creation_expression, + get_uuid_str, +) + + +def create_merging_table(spark, table_name, columns): + spark.sql( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ({columns}) USING iceberg + TBLPROPERTIES ('format-version' = '2', + 'commit.manifest.min-count-to-merge' = '1', + 'commit.manifest-merge.enabled' = 'true'); + """ + ) + + +def expire_all_snapshots_but_current(spark, table_name): + spark.sql( + f""" + CALL system.expire_snapshots( + table => '{table_name}', + older_than => TIMESTAMP '2100-01-01 00:00:00', + retain_last => 1) + """ + ) + + +def read_table(started_cluster, storage_type, table_name, query): + default_upload_directory( + started_cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + table_expression = get_creation_expression( + storage_type, + table_name, + started_cluster, + table_function=True, + ) + instance = started_cluster.instances["node1"] + return [ + instance.query( + query.format(table=table_expression), + settings={"input_format_parquet_use_native_reader_v3": use_v3}, + ) + for use_v3 in [1, 0] + ] + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_dropped_column_in_data_file(started_cluster_iceberg_with_spark, storage_type): + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_dropped_column_in_data_file_" + storage_type + "_" + get_uuid_str() + + create_merging_table(spark, TABLE_NAME, "id INT NOT NULL, legacy_col STRING") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, 'dropped');") + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP COLUMN legacy_col;") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (2);") + expire_all_snapshots_but_current(spark, TABLE_NAME) + + for result in read_table( + started_cluster_iceberg_with_spark, + storage_type, + TABLE_NAME, + "SELECT id FROM {table} ORDER BY id", + ): + assert result == "1\n2\n" + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_dropped_subcolumn_in_data_file(started_cluster_iceberg_with_spark, storage_type): + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_dropped_subcolumn_in_data_file_" + storage_type + "_" + get_uuid_str() + + create_merging_table(spark, TABLE_NAME, "id INT NOT NULL, s STRUCT") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, named_struct('a', 10, 'b', 'dropped'));") + spark.sql(f"ALTER TABLE {TABLE_NAME} DROP COLUMN s.b;") + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (2, named_struct('a', 20));") + expire_all_snapshots_but_current(spark, TABLE_NAME) + + for result in read_table( + started_cluster_iceberg_with_spark, + storage_type, + TABLE_NAME, + "SELECT id, s.a FROM {table} ORDER BY id", + ): + assert result == "1\t10\n2\t20\n" diff --git a/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.reference b/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.reference new file mode 100644 index 0000000000000..3e7b1e4deacab --- /dev/null +++ b/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.reference @@ -0,0 +1,5 @@ +1 +2 +1 +2 +ICEBERG_SPECIFICATION_VIOLATION diff --git a/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.sh b/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.sh new file mode 100755 index 0000000000000..8470f12e2f025 --- /dev/null +++ b/tests/queries/0_stateless/05137_iceberg_dropped_column_in_data_file_113324.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires IcebergLocal (USE_AVRO build option) and pyarrow. +# +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/113324 +# +# `DROP COLUMN` is a metadata-only operation, so data files written before it keep the column with +# its field id. A reader projects by field id and simply does not select a file column that the +# read schema has no id for (https://iceberg.apache.org/spec/#column-projection), but the native +# (V3) Parquet reader threw ICEBERG_SPECIFICATION_VIOLATION instead, which made such tables +# unreadable. The dropped id is at or below `last-column-id` of the table metadata, so the reader +# can tell it apart from an id the table has never assigned, which is still reported. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `y` takes field id 2 and is then dropped, so the current schema holds only `x` (field id 1) while +# `last-column-id` stays 2: field id 2 is a dropped column, field id 3 was never assigned. +create_table_with_dropped_field_id_2() { + local table=$1 + local table_dir=$2 + rm -rf "${table_dir}" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query " + CREATE TABLE ${table} (x Int32) ENGINE = IcebergLocal('${table_dir}/'); + ALTER TABLE ${table} ADD COLUMN y Nullable(Int64); + ALTER TABLE ${table} DROP COLUMN y; + INSERT INTO ${table} (x) VALUES (1), (2); + " +} + +# Rewrite the single data file so that it also holds a column with `field_id`, as a file written +# before the column was dropped does. `ghost_name` is the name that column has in the file. +add_column_to_data_file() { + local table_dir=$1 + local field_id=$2 + local ghost_name=$3 + python3 - "$(ls "${table_dir}"/data/*.parquet | head -1)" "${field_id}" "${ghost_name}" <<'PY' +import sys +import pyarrow as pa +import pyarrow.parquet as pq + +path, field_id, ghost_name = sys.argv[1], sys.argv[2].encode(), sys.argv[3] +ghost = pa.field(ghost_name, pa.int64(), nullable=True, metadata={b"PARQUET:field_id": field_id}) +x = pa.field("x", pa.int32(), nullable=False, metadata={b"PARQUET:field_id": b"1"}) +# The dropped column comes first, so a reader that matches by position rather than by field id +# would pick it up as well. +table = pa.Table.from_arrays( + [pa.array([777, 888], pa.int64()), pa.array([1, 2], pa.int32())], + schema=pa.schema([ghost, x]), +) +pq.write_table(table, path) +PY +} + +TABLE_DROPPED="t_${CLICKHOUSE_DATABASE}_dropped" +TABLE_DROPPED_DIR="${USER_FILES_PATH}/${TABLE_DROPPED}" +TABLE_SHADOWED="t_${CLICKHOUSE_DATABASE}_shadowed" +TABLE_SHADOWED_DIR="${USER_FILES_PATH}/${TABLE_SHADOWED}" +TABLE_UNASSIGNED="t_${CLICKHOUSE_DATABASE}_unassigned" +TABLE_UNASSIGNED_DIR="${USER_FILES_PATH}/${TABLE_UNASSIGNED}" + +trap ' + rm -rf "${TABLE_DROPPED_DIR}" "${TABLE_SHADOWED_DIR}" "${TABLE_UNASSIGNED_DIR}" + ${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS ${TABLE_DROPPED}; + DROP TABLE IF EXISTS ${TABLE_SHADOWED}; + DROP TABLE IF EXISTS ${TABLE_UNASSIGNED}; + " +' EXIT + +# The dropped column is not selected, and reading the table works. +create_table_with_dropped_field_id_2 "${TABLE_DROPPED}" "${TABLE_DROPPED_DIR}" +add_column_to_data_file "${TABLE_DROPPED_DIR}" 2 y +${CLICKHOUSE_CLIENT} --input_format_parquet_use_native_reader_v3=1 --query "SELECT x FROM ${TABLE_DROPPED} ORDER BY x;" + +# A column dropped and later re-added under the same name keeps its old name in the old data files, +# so the name of a dropped column may coincide with the name of a column of the read schema. The +# projection is by field id, so the values of the dropped column must not be returned for `x`. +create_table_with_dropped_field_id_2 "${TABLE_SHADOWED}" "${TABLE_SHADOWED_DIR}" +add_column_to_data_file "${TABLE_SHADOWED_DIR}" 2 x +${CLICKHOUSE_CLIENT} --input_format_parquet_use_native_reader_v3=1 --query "SELECT x FROM ${TABLE_SHADOWED} ORDER BY x;" + +# Conversely, field id 3 is above `last-column-id`, so the table cannot ever have written it: the +# file does not belong to this table, and reading it is still an error. +create_table_with_dropped_field_id_2 "${TABLE_UNASSIGNED}" "${TABLE_UNASSIGNED_DIR}" +add_column_to_data_file "${TABLE_UNASSIGNED_DIR}" 3 z +${CLICKHOUSE_CLIENT} --input_format_parquet_use_native_reader_v3=1 --query "SELECT x FROM ${TABLE_UNASSIGNED} ORDER BY x;" 2>&1 | grep -oF "ICEBERG_SPECIFICATION_VIOLATION" | head -1 From 4cba45f6389a7c6a51d4b83054eaa57746a626fd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 16:51:43 +0000 Subject: [PATCH 29/81] Update autogenerated version to 26.8.29.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 29446392b0d64..09e487cd75e42 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54541) +SET(VERSION_REVISION 54542) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 29) -SET(VERSION_GITHASH 3694fe07040369cef78c937311cbaac5f0555484) -SET(VERSION_DESCRIBE v26.8.29.1-lts) -SET(VERSION_STRING 26.8.29.1) +SET(VERSION_PATCH 30) +SET(VERSION_GITHASH 7b8b36217c67c466bd4f903c15430b04e467b25a) +SET(VERSION_DESCRIBE v26.8.30.1-lts) +SET(VERSION_STRING 26.8.30.1) # end of autochange From 89cdea13d9bcf15fee14d0da323818550403f450 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 19:01:40 +0000 Subject: [PATCH 30/81] Backport #125899 to 26.8: Apply `http_zlib_compression_level` only to zlib-based HTTP compression --- src/Core/FormatFactorySettings.h | 2 +- src/Core/Settings.cpp | 2 + src/Server/HTTPHandler.cpp | 14 ++++++- ...zlib_compression_level_only_zlib.reference | 6 +++ ...8_http_zlib_compression_level_only_zlib.sh | 38 +++++++++++++++++++ 5 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference create mode 100755 tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index be23cba6625f9..f4bb179a550f3 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -686,7 +686,7 @@ Allow data types conversion in Native input format Read data types in binary format instead of type names in Native input format )", 0) \ DECLARE(UInt64, output_format_compression_level, 3, R"( -Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when writing to table functions `file`, `url`, `hdfs`, `s3`, or `azureBlobStorage`. +Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when writing to table functions `file`, `url`, `hdfs`, `s3`, or `azureBlobStorage`. It is also applied to the compressed response to an HTTP request with a method other than `gzip` and `deflate`, for which [http_zlib_compression_level](#http_zlib_compression_level) is used. Possible values: from `1` to `22` )", 0) \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e30a69fdb179c..b27e2e54369ac 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -2515,6 +2515,8 @@ Possible values: DECLARE(Int64, http_zlib_compression_level, 3, R"( Sets the level of data compression in the response to an HTTP request if [enable_http_compression = 1](#enable_http_compression). +The setting applies only to the zlib-based methods: `gzip` and `deflate`. The level for other methods (such as `zstd`, `lz4`, `br`, `xz`) is set by [output_format_compression_level](#output_format_compression_level). + Possible values: Numbers from 1 to 12. Levels above `9` require the default build with `libdeflate`; a build without `libdeflate` supports levels 1 to 9. )", 0) \ DECLARE(SnappyMode, snappy_mode, SnappyMode::Basic, R"( diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index df830316d51b3..594bcb0580d47 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -82,6 +82,7 @@ namespace Setting extern const SettingsBool http_wait_end_of_query; extern const SettingsBool http_write_exception_in_output_format; extern const SettingsInt64 http_zlib_compression_level; + extern const SettingsUInt64 output_format_compression_level; extern const SettingsUInt64 input_format_max_block_wait_ms; extern const SettingsUInt64 readonly; extern const SettingsBool run_query_in_background; @@ -721,6 +722,15 @@ void HTTPHandler::processQuery( bool enable_http_compression = params.getParsedLast("enable_http_compression", settings[Setting::enable_http_compression]); Int64 http_zlib_compression_level = params.getParsed("http_zlib_compression_level", settings[Setting::http_zlib_compression_level]); + /// `http_zlib_compression_level` is for the zlib-based methods only. Other methods have their own ranges + /// and meanings of levels (e.g. `lz4` switches to the slow high-compression mode starting from level 2), + /// so they use the generic `output_format_compression_level`. + auto compression_level_for_method = [&](CompressionMethod method) + { + if (method == CompressionMethod::Gzip || method == CompressionMethod::Zlib) + return static_cast(http_zlib_compression_level); + return static_cast(settings[Setting::output_format_compression_level]); + }; /// HTTP `Content-Encoding: snappy` is standardized to use the snappy framing format, /// independent of the user-tunable `snappy_mode` (which controls generic `file()`/`url()` reads). auto snappy_mode = SnappyMode::Framed; @@ -739,7 +749,7 @@ void HTTPHandler::processQuery( used_output.wrap_compressed_holder = wrapWriteBufferWithCompressionMethod( used_output.out.get(), http_response_compression_method, - static_cast(http_zlib_compression_level), + compression_level_for_method(http_response_compression_method), 0, snappy_mode, DBMS_DEFAULT_BUFFER_SIZE, @@ -765,7 +775,7 @@ void HTTPHandler::processQuery( used_output.generic_compression_holder = wrapWriteBufferWithCompressionMethod( used_output.out.get(), response_compression_method, - static_cast(http_zlib_compression_level), + compression_level_for_method(response_compression_method), 0, settings[Setting::snappy_mode], DBMS_DEFAULT_BUFFER_SIZE, diff --git a/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference new file mode 100644 index 0000000000000..a01d276d82828 --- /dev/null +++ b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference @@ -0,0 +1,6 @@ +gzip http_zlib_compression_level=9: 200 100000 4999950000 +xz http_zlib_compression_level=12: 200 100000 4999950000 +zstd http_zlib_compression_level=12: 200 100000 4999950000 +lz4 http_zlib_compression_level=12: 200 100000 4999950000 +zstd ignores http_zlib_compression_level +zstd uses output_format_compression_level diff --git a/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh new file mode 100755 index 0000000000000..9ddb0896adcd2 --- /dev/null +++ b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_zlib_compression_level` applies only to the zlib-based methods (`gzip`, `deflate`). +# Other methods use `output_format_compression_level`. + +QUERY="SELECT+number+FROM+numbers(100000)" +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_response" + +function fetch() +{ + local encoding=$1 params=$2 extension=$3 + local code + code=$(${CLICKHOUSE_CURL} -sS -o "${FILE}.${extension}" -w '%{http_code}' -H "Accept-Encoding: ${encoding}" \ + "${CLICKHOUSE_URL}&enable_http_compression=1&${params}&query=${QUERY}") + echo -n "${encoding} ${params}: ${code} " + ${CLICKHOUSE_LOCAL} -q "SELECT count(), sum(toUInt64(line)) FROM file('${FILE}.${extension}', LineAsString)" +} + +function size() +{ + ${CLICKHOUSE_CURL} -sS -H "Accept-Encoding: $1" "${CLICKHOUSE_URL}&enable_http_compression=1&$2&query=${QUERY}" | wc -c +} + +fetch gzip "http_zlib_compression_level=9" gz +# Level 12 is valid for zlib with `libdeflate`, but not for `xz` (presets 0..9); it must not be passed to `xz`. +fetch xz "http_zlib_compression_level=12" xz +fetch zstd "http_zlib_compression_level=12" zst +fetch lz4 "http_zlib_compression_level=12" lz4 + +# `output_format_compression_level` controls the level of `zstd`, and `http_zlib_compression_level` does not. +[[ $(size zstd "http_zlib_compression_level=1") -eq $(size zstd "http_zlib_compression_level=9") ]] && echo "zstd ignores http_zlib_compression_level" +[[ $(size zstd "output_format_compression_level=1") -gt $(size zstd "output_format_compression_level=19") ]] && echo "zstd uses output_format_compression_level" + +rm -f "${FILE}".* From 32e4e7a7d4fdc86bd374b3ad14322b8213f7d816 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 23:10:07 +0000 Subject: [PATCH 31/81] Backport #125717 to 26.8: Keep only sizes and checksums of the files of a base backup opened for an incremental `BACKUP` --- src/Backups/BackupFactory.h | 4 ++++ src/Backups/BackupImpl.cpp | 40 +++++++++++++++++++++++++++++++++++-- src/Backups/BackupImpl.h | 3 +++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/Backups/BackupFactory.h b/src/Backups/BackupFactory.h index 8a7a0f7025289..a43f63e2ea924 100644 --- a/src/Backups/BackupFactory.h +++ b/src/Backups/BackupFactory.h @@ -55,6 +55,10 @@ class BackupFactory : boost::noncopyable bool use_same_s3_credentials_for_base_backup = false; bool use_same_password_for_base_backup = false; bool azure_attempt_to_create_container = true; + /// The backup is opened as the base of a backup being written. Such a base is only asked whether it has a file + /// with a given name, or with a given size and checksum, and for the size and checksum of a file. So it doesn't + /// keep the rest of the information about its files, and can't be read from. + bool for_file_lookups_only = false; ReadSettings read_settings; WriteSettings write_settings; #if CLICKHOUSE_CLOUD diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 74deb09197fd9..a6ffcdc7441ec 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -384,6 +384,9 @@ std::shared_ptr BackupImpl::getBaseBackupUnlocked() const } BackupFactory::CreateParams base_params = params.getCreateParamsForBaseBackup(std::move(effective_base_backup_info), archive_params.password); + /// A backup being written only looks up the sizes and checksums of the files of its base, to skip the files the base + /// already has. For a base of millions of files, the rest of the information about its files takes gigabytes. + base_params.for_file_lookups_only = (open_mode == OpenMode::WRITE); base_backup = BackupFactory::instance().createBackup(base_params); if ((open_mode == OpenMode::READ) && (base_backup_uuid != base_backup->getUUID())) @@ -753,8 +756,20 @@ void BackupImpl::readBackupMetadata() contents_seen = true; }; + /// Many files of a backup have the same size and checksum (e.g. `columns.txt` of the parts of a table), so the keys + /// are deduplicated while they are read, whenever their number doubles since the previous time: this keeps the vector + /// at about twice the number of distinct keys, however many duplicates there are. + static constexpr size_t min_num_sizes_and_checksums_to_deduplicate = 65536; + size_t num_sizes_and_checksums_after_deduplication = 0; + auto deduplicate_sizes_and_checksums = [&]() TSA_NO_THREAD_SAFETY_ANALYSIS + { + std::ranges::sort(sizes_and_checksums); + sizes_and_checksums.erase(std::ranges::unique(sizes_and_checksums).begin(), sizes_and_checksums.end()); + num_sizes_and_checksums_after_deduplication = sizes_and_checksums.size(); + }; + /// `readBackupMetadata` runs under `mutex` (TSA_REQUIRES), and `on_file` is invoked synchronously from - /// `parseMemoryNP` below while that lock is held, so the guarded members are safe to touch here. TSA cannot + /// `parse` below while that lock is held, so the guarded members are safe to touch here. TSA cannot /// see through the lambda boundary, hence the explicit suppression. handler.on_file = [&](const BackupMetadataHandler::Fields & f) TSA_NO_THREAD_SAFETY_ANALYSIS { @@ -853,7 +868,17 @@ void BackupImpl::readBackupMetadata() { file_names.emplace(info.file_name, size_and_checksum); if (info.size) - file_infos.try_emplace(size_and_checksum, std::move(info)); + { + if (params.for_file_lookups_only) + { + sizes_and_checksums.push_back(size_and_checksum); + if (sizes_and_checksums.size() + >= std::max(min_num_sizes_and_checksums_to_deduplicate, 2 * num_sizes_and_checksums_after_deduplication)) + deduplicate_sizes_and_checksums(); + } + else + file_infos.try_emplace(size_and_checksum, std::move(info)); + } } }; @@ -895,6 +920,9 @@ void BackupImpl::readBackupMetadata() if (!contents_seen) throw Exception(ErrorCodes::BACKUP_DAMAGED, "Backup {}: Metadata has no ", backup_name_for_logging); + deduplicate_sizes_and_checksums(); + sizes_and_checksums.shrink_to_fit(); + uncompressed_size = size_of_entries + metadata_in.count(); #if CLICKHOUSE_CLOUD && USE_SSL @@ -1165,6 +1193,8 @@ bool BackupImpl::fileExists(const SizeAndChecksum & size_and_checksum) const throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for writing. Something is wrong internally"); std::lock_guard lock{mutex}; + if (params.for_file_lookups_only) + return std::ranges::binary_search(sizes_and_checksums, size_and_checksum); return file_infos.contains(size_and_checksum); } @@ -1227,6 +1257,9 @@ BackupImpl::readFileImpl(const String & file_name, const SizeAndChecksum & size_ if (open_mode == OpenMode::WRITE) throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for writing. Something is wrong internally"); + if (params.for_file_lookups_only) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup {}: Opened only to look up its files, it can't be read from", backup_name_for_logging); + // Zero-sized files are not inserted into `file_infos` during metadata load, // but they are present in `file_names`. Short-circuit them here and return // an empty buffer without consulting `file_infos`. @@ -1404,6 +1437,9 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, if (open_mode == OpenMode::WRITE) throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for writing. Something is wrong internally"); + if (params.for_file_lookups_only) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup {}: Opened only to look up its files, it can't be read from", backup_name_for_logging); + if (size_and_checksum.first == 0) { /// Entry's data is empty. diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 87d8c1fb1cee0..54af93def55f1 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -185,6 +185,9 @@ class BackupImpl : public IBackup using SizeAndChecksum = std::pair; std::map file_names TSA_GUARDED_BY(mutex); /// Should be ordered alphabetically, see listFiles(). For empty files we assume checksum = 0. std::map file_infos TSA_GUARDED_BY(mutex); /// Information about files. Without empty files. + /// Instead of `file_infos` when the backup is opened only for lookups (`CreateParams::for_file_lookups_only`): the sizes + /// and checksums of its files, sorted and without duplicates. Without empty files. + std::vector sizes_and_checksums TSA_GUARDED_BY(mutex); /// object_key -> file name, only used by lightweight snapshot std::unordered_map file_object_keys TSA_GUARDED_BY(mutex); std::unordered_map lightweight_snapshot_file_infos TSA_GUARDED_BY(mutex); From 9ed00b22e33f39173c381dc17c78110539b7a896 Mon Sep 17 00:00:00 2001 From: Ye Yuan <44435593+Kryst4lskyxx@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:20:42 +0800 Subject: [PATCH 32/81] Fix native-client analyzer compatibility on 26.8 Use the connecting server's `connection_client_version_*` in the analyzer compatibility decision. Read the current `query_kind` before the check. Add a stateless native-protocol regression for versioned applications, pooled connections, and explicit choices from older initiators. Related: https://github.com/ClickHouse/ClickHouse/issues/125989 Related: https://github.com/ClickHouse/ClickHouse/pull/118624 --- src/Server/TCPHandler.cpp | 12 +- ...uted_analyzer_connection_version.reference | 19 ++ ...distributed_analyzer_connection_version.sh | 202 ++++++++++++++++++ 3 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05394_distributed_analyzer_connection_version.reference create mode 100755 tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2ce8857f4bc22..1e3febebb81e6 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2777,12 +2777,19 @@ void TCPHandler::processQuery(std::shared_ptr & state) /// Settings /// + /// Use the current query kind rather than the previous query's kind on this connection. + query_kind = state->query_context->getClientInfo().query_kind; + /// FIXME: Remove when allow_experimental_analyzer will become obsolete. /// Analyzer became Beta in 24.3 and started to be enabled by default. /// We have to disable it for ourselves to make sure we don't have different settings on - /// different servers. + /// different servers. Check the connecting server's version: `client_version_*` belongs + /// to the original application, whose version is unrelated to ClickHouse server versions. if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY - && VersionNumber(client_info.client_version_major, client_info.client_version_minor, client_info.client_version_patch) + && VersionNumber( + client_info.connection_client_version_major, + client_info.connection_client_version_minor, + client_info.connection_client_version_patch) < VersionNumber(23, 3, 0) && !passed_settings[Setting::allow_experimental_analyzer].changed) passed_settings.set("allow_experimental_analyzer", false); @@ -2794,7 +2801,6 @@ void TCPHandler::processQuery(std::shared_ptr & state) passed_settings.set("optimize_const_name_size", -1); auto settings_changes = passed_settings.changes(); - query_kind = state->query_context->getClientInfo().query_kind; if (query_kind == ClientInfo::QueryKind::INITIAL_QUERY) { /// Throw an exception if the passed settings violate the constraints. diff --git a/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.reference b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.reference new file mode 100644 index 0000000000000..7f91ed12b8b43 --- /dev/null +++ b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.reference @@ -0,0 +1,19 @@ +a 10 +b 14 + + 24 +a 10 +b 14 + + 24 +a 10 +b 14 + + 24 +a 10 +b 14 + + 24 +old_server 0 +old_server 0 +explicit_analyzer 1 diff --git a/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh new file mode 100755 index 0000000000000..1a7d931a45f1f --- /dev/null +++ b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +user="${CLICKHOUSE_TEST_UNIQUE_NAME}_user" +table="distributed_analyzer_connection_version" +proxy_dir=$(mktemp -d "${CLICKHOUSE_TMP}/analyzer_connection_version.XXXXXX") +proxy_pid= +printf '\n' > "${proxy_dir}/client.xml" + +cleanup() +{ + if [ -n "${proxy_pid}" ]; then + kill "${proxy_pid}" 2>/dev/null || true + wait "${proxy_pid}" 2>/dev/null || true + fi + rm -rf "${proxy_dir}" + ${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS ${table}_dist; DROP TABLE IF EXISTS ${table}; DROP USER IF EXISTS ${user};" +} +trap cleanup EXIT + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP TABLE IF EXISTS ${table}_dist; DROP TABLE IF EXISTS ${table}; + DROP USER IF EXISTS ${user}; + CREATE USER ${user} SETTINGS allow_experimental_analyzer = 1, apply_settings_from_server = 1; + CREATE TABLE ${table} (event_name String, event_count UInt64) ENGINE = Memory; + INSERT INTO ${table} VALUES ('a', 2), ('a', 3), ('b', 7); + GRANT SELECT ON ${CLICKHOUSE_DATABASE}.${table} TO ${user}; + GRANT REMOTE, CREATE TEMPORARY TABLE ON *.* TO ${user}; + CREATE TABLE ${table}_dist AS remote('${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_TCP},${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_TCP}', '${CLICKHOUSE_DATABASE}', '${table}', '${user}'); + GRANT SELECT ON ${CLICKHOUSE_DATABASE}.${table}_dist TO ${user}; +" + +mkfifo "${proxy_dir}/ports" +exec 3<> "${proxy_dir}/ports" +# A native client's application version is independent of its protocol revision. +# Negotiate revision 54460 (as in clickhouse-go 2.31.0) and replace only the +# application version in `Hello` and `ClientInfo`. All query/settings/data bytes are +# forwarded unchanged. One listener represents an application (`INITIAL_QUERY`), +# the other an old ClickHouse initiator (`SECONDARY_QUERY`). +python3 - "${CLICKHOUSE_HOST}" "${CLICKHOUSE_PORT_TCP}" "${proxy_dir}/ports" <<'PYTHON' & +import select +import socket +import socketserver +import sys +import threading + + +def read_exact(sock, size): + data = bytearray() + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError("Unexpected end of native packet") + data.extend(chunk) + return bytes(data) + + +def varuint(value): + data = bytearray() + while value > 127: + data.append((value & 127) | 128) + value >>= 7 + data.append(value) + return bytes(data) + + +def read_varuint(sock): + value = shift = 0 + while True: + byte = read_exact(sock, 1)[0] + value |= (byte & 127) << shift + if byte < 128: + return value + shift += 7 + + +def read_string(sock): + size = read_varuint(sock) + return varuint(size) + read_exact(sock, size) + + +def expect(value, expected): + if value != expected: + raise ValueError(f"Unexpected native field: {value!r}, expected {expected!r}") + + +class Handler(socketserver.BaseRequestHandler): + def handle(self): + client = self.request + with socket.create_connection((sys.argv[1], int(sys.argv[2])), timeout=10) as server: + major_version, minor_version = self.server.application_version + expect(read_varuint(client), 0) # Client Hello. + name = read_string(client) + for _ in range(3): + read_varuint(client) # Original major, minor, revision. + server.sendall(b"\x00" + name + varuint(major_version) + varuint(minor_version) + varuint(54460) + + b"".join(read_string(client) for _ in range(3))) + + expect(read_varuint(server), 0) # Server Hello. + name = read_string(server) + major, minor = read_varuint(server), read_varuint(server) + read_varuint(server) # Advertise the negotiated revision to the client. + client.sendall(b"\x00" + name + varuint(major) + varuint(minor) + varuint(54460) + + read_string(server) + read_string(server) + + varuint(read_varuint(server))) + server.sendall(read_string(client)) # Quota-key addendum. + + packet = read_varuint(client) + while packet == 4: # Client Ping can precede the first query. + server.sendall(varuint(packet)) + expect(read_varuint(server), 4) # Server Pong. + client.sendall(varuint(4)) + packet = read_varuint(client) + expect(packet, 1) # Query. + header = b"\x01" + read_string(client) + expect(read_exact(client, 1), b"\x01") # Initial query from clickhouse-client. + header += bytes([self.server.query_kind]) + header += b"".join(read_string(client) for _ in range(3)) + header += read_exact(client, 8) # Initial-query start time. + expect(read_exact(client, 1), b"\x01") # TCP interface. + header += b"\x01" + b"".join(read_string(client) for _ in range(3)) + for _ in range(3): + read_varuint(client) + header += varuint(major_version) + varuint(minor_version) + varuint(54460) + header += read_string(client) # Quota key. + header += varuint(read_varuint(client)) # Distributed depth. + read_varuint(client) # Application patch version. + header += varuint(0) + server.sendall(header) + + while True: + ready, _, _ = select.select([client, server], [], [], 30) + if not ready: + raise TimeoutError("Native proxy timed out") + for source in ready: + data = source.recv(65536) + if not data: + return + (server if source is client else client).sendall(data) + + +servers = [] +for query_kind, version in ((1, (2, 31)), (2, (22, 8))): + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + server.query_kind = query_kind + server.application_version = version + threading.Thread(target=server.serve_forever, daemon=True).start() + servers.append(server) +with open(sys.argv[3], "w") as ready: + ready.write(" ".join(str(server.server_address[1]) for server in servers) + "\n") +threading.Event().wait() +PYTHON +proxy_pid=$! +if ! read -r -t 30 application_port old_server_port <&3 +then + echo 'Timed out waiting for the native-protocol proxy' >&2 + exit 1 +fi +exec 3>&- + +# Two shards require aggregate-state blocks. Repeating on pooled interserver +# connections also exercises the previous-query-kind state of `TCPHandler`. +for analyzer in 1 0 +do + for attempt in 1 2 + do + ${CLICKHOUSE_CLIENT_BINARY} --config-file "${proxy_dir}/client.xml" \ + --database "${CLICKHOUSE_DATABASE}" --host 127.0.0.1 --port "${application_port}" --user "${user}" \ + --allow_experimental_analyzer "${analyzer}" --prefer_localhost_replica 0 \ + --max_distributed_connections 2 --max_threads 2 --use_query_cache 0 \ + --max_memory_usage 1073741824 --max_execution_time 10 \ + --timeout_before_checking_execution_speed 0 --max_rows_to_read 1000000 \ + --max_bytes_to_read 1000000000 --receive_timeout 10 --query " + SELECT events.event_name AS name, sum(events.event_count) AS total + FROM ${table}_dist AS events + GROUP BY 1 WITH TOTALS ORDER BY name FORMAT TSV + " + done +done + +# Use a clean client config above so runner-injected analyzer settings cannot +# turn the following implicit-setting check into an explicit-setting check. +# An actual old server, unlike a low-version application, still needs the +# compatibility fallback when it does not send an explicit analyzer setting. +for attempt in 1 2 +do + ${CLICKHOUSE_CLIENT_BINARY} --config-file "${proxy_dir}/client.xml" \ + --database "${CLICKHOUSE_DATABASE}" --host 127.0.0.1 --port "${old_server_port}" --user "${user}" \ + --receive_timeout 10 --query "SELECT 'old_server', toUInt8(getSetting('allow_experimental_analyzer'))" +done + +# Explicit settings from an older initiator must still take precedence. +${CLICKHOUSE_CLIENT_BINARY} --config-file "${proxy_dir}/client.xml" \ + --database "${CLICKHOUSE_DATABASE}" --host 127.0.0.1 --port "${old_server_port}" --user "${user}" \ + --allow_experimental_analyzer 1 --receive_timeout 10 \ + --query "SELECT 'explicit_analyzer', toUInt8(getSetting('allow_experimental_analyzer'))" From 9593dcdd5284f6986c483905c7746a51eb193cc6 Mon Sep 17 00:00:00 2001 From: Ye Yuan <44435593+Kryst4lskyxx@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:15:48 +0800 Subject: [PATCH 33/81] Keep analyzer regression FIFO separate from Bash tracing `shell_config.sh` uses file descriptor 3 for `BASH_XTRACEFD` under `clickhouse-test`. The regression reused that descriptor for the ports FIFO, so its readiness reader consumed a trace line and passed `--port +`. Let Bash allocate the FIFO descriptor. Keep tracing active through cleanup. The ClickHouse C++ change and expected query results remain unchanged. The original fixture reproduces the CI failure against the PR's compiled ARM64 release binary. The corrected fixture matches the full reference under tracing. Three runs through `clickhouse-test` pass with zero skips. The AMD sanitizer CI rerun remains pending. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=125992&sha=f7e4beb2dde1c1e81f2f0f0cb7446f98fde3fea6&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 Reported failure: https://github.com/ClickHouse/ClickHouse/pull/125992#issuecomment-6217514041 PR: https://github.com/ClickHouse/ClickHouse/pull/125992 --- .../05394_distributed_analyzer_connection_version.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh index 1a7d931a45f1f..663c7c8152198 100755 --- a/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh +++ b/tests/queries/0_stateless/05394_distributed_analyzer_connection_version.sh @@ -36,7 +36,8 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " " mkfifo "${proxy_dir}/ports" -exec 3<> "${proxy_dir}/ports" +# Let Bash allocate a descriptor: shell_config.sh uses fd 3 for CI tracing. +exec {ports_fd}<> "${proxy_dir}/ports" # A native client's application version is independent of its protocol revision. # Negotiate revision 54460 (as in clickhouse-go 2.31.0) and replace only the # application version in `Hello` and `ClientInfo`. All query/settings/data bytes are @@ -157,12 +158,12 @@ with open(sys.argv[3], "w") as ready: threading.Event().wait() PYTHON proxy_pid=$! -if ! read -r -t 30 application_port old_server_port <&3 +if ! read -r -t 30 application_port old_server_port <&"${ports_fd}" then echo 'Timed out waiting for the native-protocol proxy' >&2 exit 1 fi -exec 3>&- +exec {ports_fd}>&- # Two shards require aggregate-state blocks. Repeating on pooled interserver # connections also exercises the previous-query-kind state of `TCPHandler`. From c7b30576a9b881c0da99c294b194327cd9d9950e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 03:41:54 +0000 Subject: [PATCH 34/81] Backport #124905 to 26.8: Reject CR/LF in outgoing HTTP request lines and headers --- base/poco/Net/src/HTTPRequest.cpp | 24 ++++++++++++++ src/Backups/BackupIO_S3.cpp | 8 +++++ src/Core/YTsaurus/YTsaurusClient.cpp | 15 ++++++++- src/Core/YTsaurus/YTsaurusClient.h | 3 ++ src/Databases/DataLake/PaimonRestCatalog.cpp | 3 ++ src/Databases/DataLake/RestCatalog.cpp | 3 ++ src/Databases/DataLake/S3TablesCatalog.cpp | 4 +++ .../ObjectStorages/S3/diskSettings.cpp | 7 +++++ src/Functions/AI/AnthropicProvider.cpp | 14 +++++++-- src/Functions/AI/AnthropicProvider.h | 1 + src/Functions/AI/IAIProvider.cpp | 15 +++++++-- src/Functions/AI/IAIProvider.h | 10 +++++- src/Functions/AI/OpenAIProvider.cpp | 15 ++++++--- src/Functions/AI/OpenAIProvider.h | 2 ++ src/Functions/FunctionBaseAI.cpp | 3 +- src/Functions/aiEmbed.cpp | 6 +++- src/Functions/aiSimilarity.cpp | 6 +++- src/IO/GCPOAuth.cpp | 7 +++++ src/IO/GCPOAuth.h | 3 ++ src/IO/S3/PocoHTTPClient.cpp | 5 ++- src/IO/S3/Requests.cpp | 17 ++++++++++ src/IO/S3/Requests.h | 3 ++ src/IO/S3/copyS3File.cpp | 6 ++-- src/IO/WriteBufferFromS3.cpp | 4 +-- .../Impl/AvroConfluentSchemaRegistry.cpp | 5 ++- src/Storages/BigQuery/BigQueryClient.cpp | 4 +++ .../BigQuery/BigQueryConfiguration.cpp | 13 ++++++++ src/Storages/BigQuery/BigQueryConfiguration.h | 3 ++ src/Storages/BigQuery/StorageBigQuery.cpp | 5 +++ src/Storages/StorageURL.cpp | 14 ++++----- src/Storages/YTsaurus/StorageYTsaurus.cpp | 9 +++++- src/TableFunctions/TableFunctionBigQuery.cpp | 1 + .../05345_bigquery_header_injection.reference | 2 ++ .../05345_bigquery_header_injection.sh | 27 ++++++++++++++++ .../05346_ytsaurus_header_injection.reference | 2 ++ .../05346_ytsaurus_header_injection.sh | 28 +++++++++++++++++ ...05347_http_request_crlf_rejected.reference | 3 ++ .../05347_http_request_crlf_rejected.sh | 31 +++++++++++++++++++ .../05348_url_tracestate_crlf.reference | 2 ++ .../0_stateless/05348_url_tracestate_crlf.sh | 17 ++++++++++ ...avro_confluent_registry_url_crlf.reference | 1 + .../05349_avro_confluent_registry_url_crlf.sh | 18 +++++++++++ .../05350_ai_functions_api_key_crlf.reference | 1 + .../05350_ai_functions_api_key_crlf.sh | 26 ++++++++++++++++ 44 files changed, 366 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/05345_bigquery_header_injection.reference create mode 100755 tests/queries/0_stateless/05345_bigquery_header_injection.sh create mode 100644 tests/queries/0_stateless/05346_ytsaurus_header_injection.reference create mode 100755 tests/queries/0_stateless/05346_ytsaurus_header_injection.sh create mode 100644 tests/queries/0_stateless/05347_http_request_crlf_rejected.reference create mode 100755 tests/queries/0_stateless/05347_http_request_crlf_rejected.sh create mode 100644 tests/queries/0_stateless/05348_url_tracestate_crlf.reference create mode 100755 tests/queries/0_stateless/05348_url_tracestate_crlf.sh create mode 100644 tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.reference create mode 100755 tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.sh create mode 100644 tests/queries/0_stateless/05350_ai_functions_api_key_crlf.reference create mode 100755 tests/queries/0_stateless/05350_ai_functions_api_key_crlf.sh diff --git a/base/poco/Net/src/HTTPRequest.cpp b/base/poco/Net/src/HTTPRequest.cpp index 52b3cf2f714df..297b980d0b849 100644 --- a/base/poco/Net/src/HTTPRequest.cpp +++ b/base/poco/Net/src/HTTPRequest.cpp @@ -18,6 +18,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/Ascii.h" #include "Poco/String.h" +#include "Poco/Exception.h" using Poco::NumberFormatter; @@ -190,8 +191,31 @@ void HTTPRequest::setProxyCredentials(const std::string& scheme, const std::stri } +namespace +{ + bool hasLineBreak(const std::string& value) + { + return value.find_first_of("\r\n") != std::string::npos; + } + + /// CR or LF in the request line or in a header would let the value inject headers or a whole second request. + void checkNoLineBreaks(const HTTPRequest& request) + { + if (hasLineBreak(request.getMethod()) || hasLineBreak(request.getURI()) || hasLineBreak(request.getVersion())) + throw InvalidArgumentException("HTTP request line contains CR or LF"); + for (const auto& header : request) + { + if (hasLineBreak(header.first) || hasLineBreak(header.second)) + throw InvalidArgumentException("HTTP request header contains CR or LF", header.first); + } + } +} + + void HTTPRequest::write(std::ostream& ostr) const { + checkNoLineBreaks(*this); + ostr << _method << " " << _uri << " " << getVersion() << "\r\n"; HTTPMessage::write(ostr); ostr << "\r\n"; diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index f6875f0830474..edfa87bcad65f 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include #include +#include #include #include #include @@ -283,6 +284,13 @@ class S3BackupClientCreator /// S3 credentials. credentials_configuration.forbid_implicit_credentials = context->shouldRestrictUserQueryS3Credentials(); + /// `gcp_oauth` adds an `Authorization` header to every request, so it must obey the same filter. + if (boost::iequals(client_configuration.http_client, "gcp_oauth")) + { + HTTPHeaderEntries gcp_oauth_headers{{"Authorization", "Bearer"}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(gcp_oauth_headers); + } + auto shared_cache = S3::ClientCacheRegistry::instance().getOrCreateCacheForKey(s3_uri.endpoint, s3_uri.bucket); return S3::ClientFactory::instance().create( diff --git a/src/Core/YTsaurus/YTsaurusClient.cpp b/src/Core/YTsaurus/YTsaurusClient.cpp index 8761aaa182dbd..0544e977e79a1 100644 --- a/src/Core/YTsaurus/YTsaurusClient.cpp +++ b/src/Core/YTsaurus/YTsaurusClient.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -37,9 +38,21 @@ namespace Setting extern const SettingsUInt64 max_http_get_redirects; } +static String authorizationHeaderValue(const String & oauth_token) +{ + return fmt::format("OAuth {}", oauth_token); +} + YTsaurusClient::YTsaurusClient(ContextPtr context_, const ConnectionInfo & connection_info_) : context(context_), connection_info(connection_info_), log(getLogger("YTsaurusClient")) { + checkHeaders(context, connection_info.oauth_token); +} + +void YTsaurusClient::checkHeaders(const ContextPtr & context, const String & oauth_token) +{ + HTTPHeaderEntries headers{{"Authorization", authorizationHeaderValue(oauth_token)}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); } YTsaurusClient::YTsaurusClient(const YTsaurusClient & other) @@ -215,7 +228,7 @@ ReadBufferPtr YTsaurusClient::createQueryRWBuffer(const URI& uri, const ReadWrit /// Always use json format for input and output. {"Accept", "application/json"}, {"Content-Type", "application/json"}, - {"Authorization", fmt::format("OAuth {}", connection_info.oauth_token)}, + {"Authorization", authorizationHeaderValue(connection_info.oauth_token)}, {"X-YT-Header-Format", "yson"}, {"X-YT-Output-Format", fmt::format("{}json", output_params)}, }; diff --git a/src/Core/YTsaurus/YTsaurusClient.h b/src/Core/YTsaurus/YTsaurusClient.h index 4ea06debfbdc2..34acb019b59b9 100644 --- a/src/Core/YTsaurus/YTsaurusClient.h +++ b/src/Core/YTsaurus/YTsaurusClient.h @@ -58,6 +58,9 @@ class YTsaurusClient : private boost::noncopyable explicit YTsaurusClient(ContextPtr context_, const ConnectionInfo & connection_info_); + /// Validates the OAuth token sent in the `Authorization` header with `HTTPHeaderFilter`. + static void checkHeaders(const ContextPtr & context, const String & oauth_token); + YTsaurusClient(const YTsaurusClient & other); const ConnectionInfo & getConnectionInfo() { return connection_info; } diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index ea0207a4ee265..0244dc5a0f3f3 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,8 @@ DB::ReadWriteBufferFromHTTPPtr PaimonRestCatalog::createReadBuffer( DB::HTTPHeaderEntries request_headers(headers); const String bearer_token = createAuthHeaders(request_headers, endpoint, query_parameters_map, method); validateBearerToken(context, bearer_token); + /// The `dlf` provider adds headers built from the database settings (`dlf_access_key_id`, `region`). + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(request_headers); DB::WriteBufferFromOwnString headers_string; headers_string << "{"; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index c2a38403f8a0b..1f9fc472f213e 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1191,6 +1191,8 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( { auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); + /// The auth headers may carry a token from the OAuth server or values from the database settings. + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(result_headers); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -1786,6 +1788,7 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); const auto & context = getContext(); + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; if (!body_str.empty()) diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 734b42b95d06c..cf385dbd3a788 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -313,6 +314,8 @@ DB::ReadWriteBufferFromHTTPPtr S3TablesCatalog::createReadBuffer( auto result_headers = extractSigV4AuthHeaders(std::move(all_signed)); for (const auto & h : headers) result_headers.push_back(h); + /// The signed headers contain the access key id, region and session token from the database settings. + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(result_headers); LOG_DEBUG(log, "Requesting (SigV4): {}", url.toString()); @@ -358,6 +361,7 @@ void S3TablesCatalog::sendRequest( auto headers = extractSigV4AuthHeaders(std::move(all_signed)); headers.emplace_back("Content-Type", "application/json"); + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp index 8f5fa5494f912..8f7a9249ad6ab 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp @@ -293,6 +293,13 @@ getClient(const S3::URI & url, const S3Settings & settings, ContextPtr context, context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); + /// `gcp_oauth` adds an `Authorization` header to every request, so it must obey the same filter. + if (boost::iequals(client_configuration.http_client, "gcp_oauth")) + { + HTTPHeaderEntries gcp_oauth_headers{{"Authorization", "Bearer"}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(gcp_oauth_headers); + } + auto shared_cache = S3::ClientCacheRegistry::instance().getOrCreateCacheForKey(url.endpoint, url.bucket); return S3::ClientFactory::instance().create( diff --git a/src/Functions/AI/AnthropicProvider.cpp b/src/Functions/AI/AnthropicProvider.cpp index 426a829d11287..fb8ae2afd4e87 100644 --- a/src/Functions/AI/AnthropicProvider.cpp +++ b/src/Functions/AI/AnthropicProvider.cpp @@ -31,6 +31,15 @@ AnthropicProvider::AnthropicProvider(const String & endpoint_, const String & ap "Unsupported Anthropic API version '{}'. Supported: '{}'", api_version, DEFAULT_ANTHROPIC_API_VERSION); } +HTTPHeaderEntries AnthropicProvider::getHeaders() const +{ + HTTPHeaderEntries headers; + if (!api_key.empty()) /// not all providers need API key + headers.emplace_back("x-api-key", api_key); + headers.emplace_back("anthropic-version", api_version); + return headers; +} + AIResponse AnthropicProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; @@ -80,9 +89,8 @@ AIResponse AnthropicProvider::call(const AIRequest & ai_request, const Connectio Poco::Net::HTTPRequest http_request(Poco::Net::HTTPRequest::HTTP_POST, uri.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); http_request.setContentType("application/json"); - if (!api_key.empty()) /// not all providers need API key - http_request.set("x-api-key", api_key); - http_request.set("anthropic-version", api_version); + for (const auto & header : getHeaders()) + http_request.set(header.name, header.value); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); diff --git a/src/Functions/AI/AnthropicProvider.h b/src/Functions/AI/AnthropicProvider.h index 60980b71fb546..cd91752b31cc9 100644 --- a/src/Functions/AI/AnthropicProvider.h +++ b/src/Functions/AI/AnthropicProvider.h @@ -23,6 +23,7 @@ class AnthropicProvider : public IAIProvider AnthropicProvider(const String & endpoint_, const String & api_key_, const String & api_version_); AIResponse call(const AIRequest & request, const ConnectionTimeouts & timeouts) override; + HTTPHeaderEntries getHeaders() const override; private: const String endpoint; diff --git a/src/Functions/AI/IAIProvider.cpp b/src/Functions/AI/IAIProvider.cpp index 6903b803f926b..d88ec81183c2d 100644 --- a/src/Functions/AI/IAIProvider.cpp +++ b/src/Functions/AI/IAIProvider.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -67,15 +69,22 @@ AIEmbeddingResponse IAIProvider::embed(const AIEmbeddingRequest & /*ai_embedding throw Exception(ErrorCodes::NOT_IMPLEMENTED, "This AI provider does not support embeddings"); } -AIProviderPtr createAIProvider(const String & provider_name, const String & endpoint, const String & api_key, const String & api_version) +AIProviderPtr createAIProvider( + const ContextPtr & context, const String & provider_name, const String & endpoint, const String & api_key, const String & api_version) { + AIProviderPtr provider; if (provider_name == "openai" || provider_name == "huggingface" || provider_name == "tei") - return std::make_unique(endpoint, api_key); + provider = std::make_unique(endpoint, api_key); else if (provider_name == "anthropic") - return std::make_unique(endpoint, api_key, api_version); + provider = std::make_unique(endpoint, api_key, api_version); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown AI provider '{}'. Supported: 'openai', 'anthropic', 'huggingface', 'tei'", provider_name); + + /// Rejects forbidden headers and CR/LF in values (the key comes from a named collection). + auto headers = provider->getHeaders(); + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); + return provider; } } diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index c9954a1ffc1de..850218e093347 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -143,6 +145,9 @@ class IAIProvider /// Send a chat completion request and return the parsed response. virtual AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) = 0; + /// Provider-specific headers (credentials, API version) sent with every request. + virtual HTTPHeaderEntries getHeaders() const = 0; + /// Whether this provider exposes an embeddings endpoint. virtual bool supportsEmbeddings() const { return false; } @@ -152,7 +157,10 @@ class IAIProvider using AIProviderPtr = std::unique_ptr; -AIProviderPtr createAIProvider(const String & provider_name, const String & endpoint, const String & api_key, const String & api_version); +/// Also validates the provider's headers with `HTTPHeaderFilter`. +AIProviderPtr createAIProvider( + const ContextPtr & context, + const String & provider_name, const String & endpoint, const String & api_key, const String & api_version); /// Build an error message from a provider's non-200 HTTP response, for use in an exception that is logged. String formatProviderError(int status_code, const String & response_body); diff --git a/src/Functions/AI/OpenAIProvider.cpp b/src/Functions/AI/OpenAIProvider.cpp index 05d9dd508a20c..c307b2cfa6a16 100644 --- a/src/Functions/AI/OpenAIProvider.cpp +++ b/src/Functions/AI/OpenAIProvider.cpp @@ -26,6 +26,13 @@ OpenAIProvider::OpenAIProvider(const String & endpoint_, const String & api_key_ { } +HTTPHeaderEntries OpenAIProvider::getHeaders() const +{ + if (api_key.empty()) /// not all providers need API key + return {}; + return {{"Authorization", "Bearer " + api_key}}; +} + AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; @@ -61,8 +68,8 @@ AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTi Poco::Net::HTTPRequest http_request(Poco::Net::HTTPRequest::HTTP_POST, uri.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); http_request.setContentType("application/json"); - if (!api_key.empty()) /// not all providers need API key - http_request.set("Authorization", "Bearer " + api_key); + for (const auto & header : getHeaders()) + http_request.set(header.name, header.value); chassert(!ai_request.function_name.empty()); http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); http_request.setContentLength(body.size()); @@ -172,8 +179,8 @@ AIEmbeddingResponse OpenAIProvider::embed(const AIEmbeddingRequest & ai_embeddin Poco::Net::HTTPRequest http_request(Poco::Net::HTTPRequest::HTTP_POST, uri.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); http_request.setContentType("application/json"); - if (!api_key.empty()) /// not all providers need API key - http_request.set("Authorization", "Bearer " + api_key); + for (const auto & header : getHeaders()) + http_request.set(header.name, header.value); chassert(!ai_embedding_request.function_name.empty()); http_request.set("X-ClickHouse-AI-Function", ai_embedding_request.function_name); http_request.setContentLength(body.size()); diff --git a/src/Functions/AI/OpenAIProvider.h b/src/Functions/AI/OpenAIProvider.h index 771c4e0a9522a..38d65fc6b8138 100644 --- a/src/Functions/AI/OpenAIProvider.h +++ b/src/Functions/AI/OpenAIProvider.h @@ -34,6 +34,8 @@ class OpenAIProvider : public IAIProvider OpenAIProvider(const String & endpoint_, const String & api_key_); AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) override; + HTTPHeaderEntries getHeaders() const override; + bool supportsEmbeddings() const override { return true; } AIEmbeddingResponse embed(const AIEmbeddingRequest & ai_embedding_request, const ConnectionTimeouts & timeouts) override; diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 8884b38835f6a..7aea1b1826401 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -466,7 +466,8 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, checkSanityBeforeExecuteImpl(arguments, result_type, input_rows_count); String system_prompt = sanitizeForModel(buildSystemPrompt(arguments, params)); auto response_format = buildResponseFormat(arguments); - auto provider = createAIProvider(params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); + auto provider = createAIProvider( + getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (input_rows_count == 0) return result_type->createColumn(); diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index 005cdbf1540f3..26cb68c767f6c 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -107,7 +107,11 @@ class FunctionAiEmbed final : public IFunction String model(arguments[model_arg_index].column->getDataAt(0)); auto provider = createAIProvider( - params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); + getContext(), + params.collection.provider, + params.collection.endpoint, + params.collection.api_key, + params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index 22ec7b2a5c0b4..7c35c9bf94bd6 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -135,7 +135,11 @@ class FunctionAiSimilarity final : public IFunction String model(arguments[model_arg_index].column->getDataAt(0)); auto provider = createAIProvider( - params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); + getContext(), + params.collection.provider, + params.collection.endpoint, + params.collection.api_key, + params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); diff --git a/src/IO/GCPOAuth.cpp b/src/IO/GCPOAuth.cpp index cc719adbe9c83..25621d499332f 100644 --- a/src/IO/GCPOAuth.cpp +++ b/src/IO/GCPOAuth.cpp @@ -91,6 +91,7 @@ GCPOAuthToken postTokenRequest( GCPOAuthToken result; result.access_token = object->getValue("access_token"); + checkGCPOAuthAccessToken(result.access_token); if (object->has("expires_in")) result.expires_in = object->getValue("expires_in"); @@ -99,6 +100,12 @@ GCPOAuthToken postTokenRequest( } +void checkGCPOAuthAccessToken(const std::string & access_token) +{ + if (access_token.find_first_of("\r\n") != std::string::npos) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "GCP access token contains CR or LF"); +} + GCPOAuthToken fetchGCPOAuthToken( const std::string & client_id, const std::string & client_secret, diff --git a/src/IO/GCPOAuth.h b/src/IO/GCPOAuth.h index ce6f61b29d07d..55653178f128f 100644 --- a/src/IO/GCPOAuth.h +++ b/src/IO/GCPOAuth.h @@ -14,6 +14,9 @@ struct GCPOAuthToken Int64 expires_in = 3600; /// seconds until expiry as reported by the token endpoint }; +/// Throws if the access token from a token endpoint cannot be sent in an `Authorization` header. +void checkGCPOAuthAccessToken(const std::string & access_token); + /// Exchange a Google OAuth2 refresh token for an access token by POSTing to /// https://oauth2.googleapis.com/token. /// diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 5e3f11aec9739..303afead2c090 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -924,9 +924,12 @@ PocoHTTPClientGCPOAuth::BearerToken PocoHTTPClientGCPOAuth::requestBearerToken() throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Unexpected structure of response. Expected Bearer token, got {}", token_type); + auto token = object->getValue("access_token"); + checkGCPOAuthAccessToken(token); + return { - .token = object->getValue("access_token"), + .token = std::move(token), .is_valid_to = std::chrono::system_clock::now() + std::chrono::seconds(object->getValue("expires_in")) }; } diff --git a/src/IO/S3/Requests.cpp b/src/IO/S3/Requests.cpp index 763eeda8a5674..4caf5ce6a3ba4 100644 --- a/src/IO/S3/Requests.cpp +++ b/src/IO/S3/Requests.cpp @@ -2,6 +2,7 @@ #if USE_AWS_S3 +#include #include #include #include @@ -10,9 +11,25 @@ #include #include +namespace DB +{ +namespace ErrorCodes +{ + extern const int INVALID_SETTING_VALUE; +} +} + namespace DB::S3 { +Model::StorageClass getStorageClassForName(const std::string & name) +{ + /// An unknown name is sent as is, so it must not break the header line. + if (name.find_first_of("\r\n") != std::string::npos) + throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting storage_class has invalid value: it contains CR or LF"); + return Model::StorageClassMapper::GetStorageClassForName(name); +} + Aws::Http::HeaderValueCollection CopyObjectRequest::GetRequestSpecificHeaders() const { auto headers = Model::CopyObjectRequest::GetRequestSpecificHeaders(); diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 90134daea2386..a7de2092a5d62 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -61,6 +61,9 @@ inline void setChecksumAlgorithm(R & request) } }; +/// The `x-amz-storage-class` value for a storage class name; throws if the name contains CR or LF. +Model::StorageClass getStorageClassForName(const std::string & name); + template class ExtendedRequest : public BaseRequest { diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 73143fbaf52e8..bbc2fdb8db3cf 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -144,7 +144,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); client_ptr->setKMSHeaders(request); } @@ -481,7 +481,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); @@ -712,7 +712,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 2dbfc8d0f9fdf..2ff3d42c92533 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -421,7 +421,7 @@ void WriteBufferFromS3::createMultipartUpload() /// The storage class of a multipart-uploaded object is determined by the CreateMultipartUpload /// request; it cannot be set on UploadPart or CompleteMultipartUpload. See issue #68551. if (!request_settings[S3RequestSetting::storage_class_name].value.empty()) - req.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); + req.SetStorageClass(S3::getStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); client_ptr->setKMSHeaders(req); @@ -722,7 +722,7 @@ S3::PutObjectRequest WriteBufferFromS3::getPutRequest(PartData & data) if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); if (!request_settings[S3RequestSetting::storage_class_name].value.empty()) - req.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); + req.SetStorageClass(S3::getStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); if (!write_settings.object_storage_write_if_none_match.empty()) req.SetIfNoneMatch(write_settings.object_storage_write_if_none_match); diff --git a/src/Processors/Formats/Impl/AvroConfluentSchemaRegistry.cpp b/src/Processors/Formats/Impl/AvroConfluentSchemaRegistry.cpp index 1c359083fa33e..bcbe248d46bf8 100644 --- a/src/Processors/Formats/Impl/AvroConfluentSchemaRegistry.cpp +++ b/src/Processors/Formats/Impl/AvroConfluentSchemaRegistry.cpp @@ -264,7 +264,10 @@ uint32_t ConfluentSchemaRegistry::registerSchema( /// Build the request path directly to preserve percent-encoding. /// Poco::URI normalizes paths by decoding %2F -> /, which breaks /// subjects containing slashes. - std::string request_path = base_url.getPath() + "/subjects/" + encoded_subject + "/versions"; + /// `getPath` returns the decoded path, so encode it again like `getPathAndQuery` does for the schema fetch. + std::string request_path; + Poco::URI::encode(base_url.getPath(), "?#", request_path); + request_path += "/subjects/" + encoded_subject + "/versions"; LOG_TRACE(getLogger("ConfluentSchemaRegistry"), "Registering schema under subject '{}' at path {}", subject, request_path); auto connection_timeouts = buildTimeouts(timeouts); diff --git a/src/Storages/BigQuery/BigQueryClient.cpp b/src/Storages/BigQuery/BigQueryClient.cpp index 15ce6d79cbfe4..65317e1ccc1b1 100644 --- a/src/Storages/BigQuery/BigQueryClient.cpp +++ b/src/Storages/BigQuery/BigQueryClient.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #if USE_SSL @@ -226,6 +227,9 @@ Poco::JSON::Object::Ptr BigQueryClient::requestJSON( out_stream_callback = [&request_body](std::ostream & os) { os << request_body; }; } + /// Also covers tokens returned by the OAuth endpoint and tables attached without the CREATE check. + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); + auto buf = BuilderRWBufferFromHTTP(uri) .withConnectionGroup(HTTPConnectionGroupType::HTTP) .withMethod(method) diff --git a/src/Storages/BigQuery/BigQueryConfiguration.cpp b/src/Storages/BigQuery/BigQueryConfiguration.cpp index 981b3ae455b2c..5eec01926a17e 100644 --- a/src/Storages/BigQuery/BigQueryConfiguration.cpp +++ b/src/Storages/BigQuery/BigQueryConfiguration.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include #include @@ -7,6 +9,7 @@ #include #include #include +#include #include #include @@ -217,4 +220,14 @@ BigQueryConfiguration BigQueryConfiguration::fromArguments(ASTs & args, ContextP return configuration; } +void BigQueryConfiguration::checkHeaders(const ContextPtr & context) const +{ + HTTPHeaderEntries headers; + if (!access_token.empty()) + headers.emplace_back("Authorization", "Bearer " + access_token); + if (!billing_project.empty()) + headers.emplace_back("X-Goog-User-Project", billing_project); + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); +} + } diff --git a/src/Storages/BigQuery/BigQueryConfiguration.h b/src/Storages/BigQuery/BigQueryConfiguration.h index 031e0764a1e48..5939155b4c70c 100644 --- a/src/Storages/BigQuery/BigQueryConfiguration.h +++ b/src/Storages/BigQuery/BigQueryConfiguration.h @@ -57,6 +57,9 @@ struct BigQueryConfiguration /// When `table_id` is provided (the persistent table engine), the table is registered as a dependency /// of the named collection so that `DROP NAMED COLLECTION` is blocked while the table exists. static BigQueryConfiguration fromArguments(ASTs & args, ContextPtr context, const StorageID * table_id = nullptr); + + /// Validates the user-supplied values sent as HTTP headers with `HTTPHeaderFilter`. + void checkHeaders(const ContextPtr & context) const; }; } diff --git a/src/Storages/BigQuery/StorageBigQuery.cpp b/src/Storages/BigQuery/StorageBigQuery.cpp index 80dee16aa76ff..8e986a21bc360 100644 --- a/src/Storages/BigQuery/StorageBigQuery.cpp +++ b/src/Storages/BigQuery/StorageBigQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -656,6 +657,10 @@ void registerStorageBigQuery(StorageFactory & factory) [](const StorageFactory::Arguments & args) { auto configuration = StorageBigQuery::getConfiguration(args.engine_args, args.getLocalContext(), &args.table_id); + /// Skipped for a stored definition so that a forbidden header does not block server startup; + /// requests still check it. + if (isFreshTableDefinition(args.mode, args.query.attach_short_syntax) && !args.getLocalContext()->isRecoveryFromStoredMetadata()) + configuration.checkHeaders(args.getLocalContext()); return std::make_shared( args.table_id, std::move(configuration), args.columns, args.constraints, args.comment, args.getLocalContext()); }, diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index fca4d854d2800..4874528cc4035 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -269,7 +269,7 @@ IStorageURLBase::IStorageURLBase( namespace { - HTTPHeaderEntries getHeaders(const HTTPHeaderEntries & headers_) + HTTPHeaderEntries getHeaders(const HTTPHeaderEntries & headers_, const ContextPtr & context) { HTTPHeaderEntries headers(headers_.begin(), headers_.end()); @@ -277,12 +277,12 @@ namespace const auto & current_trace_context = OpenTelemetry::CurrentContext(); if (current_trace_context.isTraceEnabled()) { - headers.emplace_back("traceparent", current_trace_context.composeTraceparentHeader()); - + /// The user headers are checked in the `StorageURL` constructor; `tracestate` comes from the client as is. + HTTPHeaderEntries trace_headers{{"traceparent", current_trace_context.composeTraceparentHeader()}}; if (!current_trace_context.tracestate.empty()) - { - headers.emplace_back("tracestate", current_trace_context.tracestate); - } + trace_headers.emplace_back("tracestate", current_trace_context.tracestate); + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(trace_headers); + headers.insert(headers.end(), trace_headers.begin(), trace_headers.end()); } return headers; @@ -387,7 +387,7 @@ StorageURLSource::StorageURLSource( , format_settings(format_settings_) , parser_shared_resources(std::move(parser_shared_resources_)) , format_filter_info(std::move(format_filter_info_)) - , headers(getHeaders(headers_)) + , headers(getHeaders(headers_, context_)) , need_only_count(need_only_count_) , storage_id(std::move(storage_id_)) , hive_partition_columns_to_read_from_file_path(info.hive_partition_columns_to_read_from_file_path) diff --git a/src/Storages/YTsaurus/StorageYTsaurus.cpp b/src/Storages/YTsaurus/StorageYTsaurus.cpp index fcf5d0b347d2e..adb25d6e4a578 100644 --- a/src/Storages/YTsaurus/StorageYTsaurus.cpp +++ b/src/Storages/YTsaurus/StorageYTsaurus.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -159,9 +160,15 @@ void registerStorageYTsaurus(StorageFactory & factory) if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_ytsaurus_table_engine]) throw Exception(ErrorCodes::UNKNOWN_STORAGE, "Table engine YTsaurus is experimental. " "Set `allow_experimental_ytsaurus_table_engine` setting to enable it"); + auto configuration = StorageYTsaurus::getConfiguration( + args.engine_args, YTsaurusSettings::createFromQuery(*args.storage_def), args.getLocalContext(), &args.table_id); + /// Skipped for a stored definition so that a forbidden header does not block server startup; + /// the client still checks it. + if (isFreshTableDefinition(args.mode, args.query.attach_short_syntax) && !args.getLocalContext()->isRecoveryFromStoredMetadata()) + YTsaurusClient::checkHeaders(args.getLocalContext(), configuration.oauth_token); return std::make_shared( args.table_id, - StorageYTsaurus::getConfiguration(args.engine_args, YTsaurusSettings::createFromQuery(*args.storage_def), args.getLocalContext(), &args.table_id), + std::move(configuration), args.columns, args.constraints, args.comment); diff --git a/src/TableFunctions/TableFunctionBigQuery.cpp b/src/TableFunctions/TableFunctionBigQuery.cpp index 9db7b9cbe3e79..f0de4c2681447 100644 --- a/src/TableFunctions/TableFunctionBigQuery.cpp +++ b/src/TableFunctions/TableFunctionBigQuery.cpp @@ -95,6 +95,7 @@ void TableFunctionBigQuery::parseArguments(const ASTPtr & ast_function, ContextP throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function 'bigquery' must have arguments"); configuration = std::make_shared(BigQueryConfiguration::fromArguments(func_args.arguments->children, context)); + configuration->checkHeaders(context); /// The token provider only stores the configuration here; no network request is made until a token /// is actually needed (during schema inference or execution). token_provider = std::make_shared(*configuration); diff --git a/tests/queries/0_stateless/05345_bigquery_header_injection.reference b/tests/queries/0_stateless/05345_bigquery_header_injection.reference new file mode 100644 index 0000000000000..c3d4f625310cc --- /dev/null +++ b/tests/queries/0_stateless/05345_bigquery_header_injection.reference @@ -0,0 +1,2 @@ +0 +OK diff --git a/tests/queries/0_stateless/05345_bigquery_header_injection.sh b/tests/queries/0_stateless/05345_bigquery_header_injection.sh new file mode 100755 index 0000000000000..ed3d74bffa92f --- /dev/null +++ b/tests/queries/0_stateless/05345_bigquery_header_injection.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Values that become HTTP header values must not contain CR or LF. +# The base_url points to a closed port, so an unchecked value fails with a network error instead. +# A full-definition ATTACH is user input like CREATE; Atomic and Replicated databases need a UUID for it. +UUID_CLAUSE="" +if [ "$($CLICKHOUSE_CLIENT -q "SELECT engine FROM system.databases WHERE name = currentDatabase()")" != "Ordinary" ]; then + UUID_CLAUSE="UUID '$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()")'" +fi + +$CLICKHOUSE_CLIENT -m <&1 \ + | grep -o -m1 "HTTP request header contains CR or LF" + +# An unknown storage class is sent as is in `x-amz-storage-class` when `s3_validate_request_settings = 0`, +# so CR or LF in it must be rejected before the request is made. +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('http://localhost:${CLICKHOUSE_PORT_HTTP}/bucket/${CLICKHOUSE_DATABASE}.csv', 'key', 'secret', 'CSV', 'x UInt8', + storage_class_name = 'STANDARD\r\nX-Injected: 1') + SETTINGS s3_validate_request_settings = 0, s3_truncate_on_insert = 1 + SELECT 1" 2>&1 | grep -o -m1 "Setting storage_class has invalid value: it contains CR or LF" + +$CLICKHOUSE_CLIENT -m -q " + DROP TABLE IF EXISTS t_storage_class; + CREATE TABLE t_storage_class (x UInt8) ENGINE = MergeTree ORDER BY x; + INSERT INTO t_storage_class SELECT 1; +" +$CLICKHOUSE_CLIENT -q " + BACKUP TABLE t_storage_class TO S3('http://localhost:${CLICKHOUSE_PORT_HTTP}/bucket/${CLICKHOUSE_DATABASE}_backup', 'key', 'secret') + SETTINGS s3_storage_class = 'STANDARD\r\nX-Injected: 1'" 2>&1 | grep -o -m1 "Setting storage_class has invalid value: it contains CR or LF" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_storage_class" diff --git a/tests/queries/0_stateless/05348_url_tracestate_crlf.reference b/tests/queries/0_stateless/05348_url_tracestate_crlf.reference new file mode 100644 index 0000000000000..0dd911d458ede --- /dev/null +++ b/tests/queries/0_stateless/05348_url_tracestate_crlf.reference @@ -0,0 +1,2 @@ +HTTP header "tracestate" has invalid character +1 diff --git a/tests/queries/0_stateless/05348_url_tracestate_crlf.sh b/tests/queries/0_stateless/05348_url_tracestate_crlf.sh new file mode 100755 index 0000000000000..c05c60ab14cee --- /dev/null +++ b/tests/queries/0_stateless/05348_url_tracestate_crlf.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The client's OpenTelemetry `tracestate` is forwarded by `url` as an HTTP header, so CR or LF in it must be rejected. +$CLICKHOUSE_CLIENT \ + --opentelemetry-traceparent "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" \ + --opentelemetry-tracestate $'a=b\r\nX-Injected: 1' \ + -q "SELECT * FROM url('http://localhost:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'TSV', 'x UInt8')" 2>&1 \ + | grep -o -m1 'HTTP header "tracestate" has invalid character' + +$CLICKHOUSE_CLIENT \ + --opentelemetry-traceparent "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" \ + --opentelemetry-tracestate 'a=b' \ + -q "SELECT * FROM url('http://localhost:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'TSV', 'x UInt8')" diff --git a/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.reference b/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.reference new file mode 100644 index 0000000000000..77f58b898faf2 --- /dev/null +++ b/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.reference @@ -0,0 +1 @@ +remote server /x%0D%0AX-Injected:%201/subjects/s/versions diff --git a/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.sh b/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.sh new file mode 100755 index 0000000000000..e967ffe50b27d --- /dev/null +++ b/tests/queries/0_stateless/05349_avro_confluent_registry_url_crlf.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs Avro + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The path of `format_avro_schema_registry_url` must stay percent-encoded in the schema registration +# request, so an encoded CR or LF cannot reach the request line. The server answers with an error +# that echoes the requested path. The default 1 second timeouts are too short for slow builds. +${CLICKHOUSE_CLIENT} --query " + SELECT 1::UInt8 AS a FORMAT AvroConfluent + SETTINGS format_avro_schema_registry_url = 'http://localhost:${CLICKHOUSE_PORT_HTTP}/x%0D%0AX-Injected:%201', + output_format_avro_confluent_subject = 's', format_avro_schema_registry_max_retries = 0, + format_avro_schema_registry_connection_timeout = 60, format_avro_schema_registry_send_timeout = 60, + format_avro_schema_registry_receive_timeout = 60" 2>&1 \ + | grep -o -m1 "remote server /[^ ]*/versions" diff --git a/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.reference b/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.sh b/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.sh new file mode 100755 index 0000000000000..23e3dc2feb1b1 --- /dev/null +++ b/tests/queries/0_stateless/05350_ai_functions_api_key_crlf.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The `api_key` of an AI named collection is sent as an HTTP header, so CR or LF in it must be rejected +# before any request. Port 1 refuses at once, so an unchecked key fails with a network error instead. +OPENAI="ai_crlf_openai_${CLICKHOUSE_DATABASE}" +ANTHROPIC="ai_crlf_anthropic_${CLICKHOUSE_DATABASE}" +trap '$CLICKHOUSE_CLIENT -m -q "DROP NAMED COLLECTION IF EXISTS ${OPENAI}; DROP NAMED COLLECTION IF EXISTS ${ANTHROPIC};"' EXIT + +$CLICKHOUSE_CLIENT -m -q " + CREATE NAMED COLLECTION ${OPENAI} AS + provider = 'openai', endpoint = 'http://127.0.0.1:1/v1/chat/completions', model = 'm', api_key = 'key\r\nX-Injected: 1'; + CREATE NAMED COLLECTION ${ANTHROPIC} AS + provider = 'anthropic', endpoint = 'http://127.0.0.1:1/v1/messages', model = 'm', api_key = 'key\nX-Injected: 1'; +" + +$CLICKHOUSE_CLIENT -m -q " + SET ai_function_max_retries = 0; + SELECT aiGenerate('hi', map('credentials', '${OPENAI}')); -- { serverError BAD_ARGUMENTS } + SELECT aiEmbed('hi', 'm', map('credentials', '${OPENAI}')); -- { serverError BAD_ARGUMENTS } + SELECT aiGenerate('hi', map('credentials', '${ANTHROPIC}')); -- { serverError BAD_ARGUMENTS } + SELECT 'OK'; +" From f8238d11d24b1a88c4720513c938164d7ed7a000 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 15:56:06 +0000 Subject: [PATCH 35/81] Backport #118866 to 26.8: Require SELECT on the source when CREATE TABLE AS inherits credentials --- src/Interpreters/InterpreterCreateQuery.cpp | 84 +++++++++++++++++++ ..._external_engine_requires_select.reference | 23 +++++ ...eate_as_external_engine_requires_select.sh | 73 ++++++++++++++++ ..._settings_secret_requires_select.reference | 6 ++ ..._engine_settings_secret_requires_select.sh | 43 ++++++++++ ..._on_cluster_inherits_credentials.reference | 17 ++++ ...eate_as_on_cluster_inherits_credentials.sh | 69 +++++++++++++++ ..._series_inner_engine_credentials.reference | 6 ++ ...as_time_series_inner_engine_credentials.sh | 45 ++++++++++ ...te_as_secret_key_requires_select.reference | 8 ++ ...32_create_as_secret_key_requires_select.sh | 44 ++++++++++ ...cit_engine_requires_show_columns.reference | 2 + ...r_explicit_engine_requires_show_columns.sh | 28 +++++++ 13 files changed, 448 insertions(+) create mode 100644 tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference create mode 100755 tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh create mode 100644 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference create mode 100755 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh create mode 100644 tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference create mode 100755 tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh create mode 100644 tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference create mode 100755 tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh create mode 100644 tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference create mode 100755 tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh create mode 100644 tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference create mode 100755 tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 84344019e717a..97931dfc0803f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1318,6 +1318,8 @@ namespace { *ptr = nullptr; }); + /// children keeps the old nodes, and hasSecretParts reads them + storage.children.clear(); auto engine_ast = make_intrusive(); engine_ast->name = "Null"; @@ -1464,12 +1466,24 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const String as_database_name = getContext()->resolveDatabase(create.as_database); String as_table_name = create.as_table; + /// check SHOW COLUMNS first, else the error tells a user who cannot see the table that it + /// holds credentials + getContext()->checkAccess(AccessType::SHOW_COLUMNS, as_database_name, as_table_name); + ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(as_table_name, getContext()); const auto & as_create = as_create_ptr->as(); const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); + /// SHOW CREATE TABLE masks credentials, so a copy of them gives the source data to a user who + /// cannot SELECT it. SHOW COLUMNS already shows the rest + auto check_access_to_inherited_definition = [&](const IAST & definition) + { + if (definition.hasSecretParts()) + getContext()->checkAccess(AccessType::SELECT, as_database_name, as_table_name); + }; + if (as_create.is_ordinary_view) throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name); @@ -1498,6 +1512,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const /// clauses were specified for the new table; otherwise keep the explicit storage definition. if (!create.storage) { + check_access_to_inherited_definition(*as_create.as_table_function); create.set(create.as_table_function, as_create.as_table_function->ptr()); return; } @@ -1506,6 +1521,16 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { storage_def = boost::static_pointer_cast(as_create.storage->ptr()); create.is_time_series_table = as_create.is_time_series_table; + + /// normalizeTimeSeriesDefinition copies the settings and the inner engines of the source, + /// and this query inherits each one it does not give itself + if (as_create.is_time_series_table && as_create.storage->settings + && !(create.storage && create.storage->settings)) + check_access_to_inherited_definition(*as_create.storage->settings); + if (as_create.is_time_series_table && as_create.targets) + for (const auto & target : as_create.targets->targets) + if (target.inner_engine && !create.getTargetInnerEngine(target.kind)) + check_access_to_inherited_definition(*target.inner_engine); } else { @@ -1513,6 +1538,36 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const } } + if (storage_def) + { + /// check what the server stores: settings in this query replace the source settings, and + /// Null can replace an external engine + auto inherited = boost::static_pointer_cast(storage_def->clone()); + if (create.storage) + { + /// inheritStorageFromSource keeps the clauses this query gives, so drop them here too + if (create.storage->partition_by) + inherited->reset(inherited->partition_by); + if (create.storage->primary_key) + inherited->reset(inherited->primary_key); + if (create.storage->order_by) + inherited->reset(inherited->order_by); + if (create.storage->sample_by) + inherited->reset(inherited->sample_by); + if (create.storage->ttl_table) + inherited->reset(inherited->ttl_table); + if (create.storage->unique_key) + inherited->reset(inherited->unique_key); + if (inherited->settings && create.storage->settings) + for (const auto & change : create.storage->settings->changes) + inherited->settings->changes.removeSetting(change.name); + } + replaceExternalEngineWithNullIfNeeded( + *inherited, getContext()->getSettingsRef()[Setting::restore_replace_external_engines_to_null]); + + check_access_to_inherited_definition(*inherited); + } + if (create.storage) { /// This table already has a (possibly partial) storage definition. @@ -3463,6 +3518,35 @@ BlockIO InterpreterCreateQuery::execute() && create.storage->engine->name == "Backup" && create.storage->engine->arguments) DatabaseBackup::parseAndAuthorizeLocator(create.storage->engine->arguments->children, getContext()); + /// this branch returns before setEngine, and the worker builds AS src with no user, so + /// check the inherited definition here, on a copy of the query + if (!create.as_table.empty()) + { + /// a worker resolves a name without a database in its own database, so set ours, as + /// the UUIDs above do + create.as_database = getContext()->resolveDatabase(create.as_database); + + if (create.storage && create.storage->engine) + getContext()->checkAccess(AccessType::SHOW_COLUMNS, create.as_database, create.as_table); + + /// OLDEST_VERSION sends no settings, so a worker replaces nothing with Null. check + /// what the worker builds, not what our settings give + auto preflight_context = Context::createCopy(getContext()); + if (on_cluster_version == DDLLogEntry::OLDEST_VERSION) + { + preflight_context->setSetting("restore_replace_external_engines_to_null", false); + preflight_context->setSetting("restore_replace_external_table_functions_to_null", false); + } + + ASTPtr inherited_query = query_ptr->clone(); + auto & inherited = inherited_query->as(); + InterpreterCreateQuery(inherited_query, preflight_context).setEngine(inherited); + + /// getRequiredAccess sees no engine, because this query holds none of its own + if (inherited.storage && inherited.storage->engine) + getContext()->checkAccess(AccessType::TABLE_ENGINE, inherited.storage->engine->name); + } + return executeQueryOnCluster(create); } } diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference new file mode 100644 index 0000000000000..f0edc2dad6cfa --- /dev/null +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference @@ -0,0 +1,23 @@ +with SHOW COLUMNS only: +-- copy_of_local_src: +MergeTree +-- copy_of_url_src: +necessary to have the grant SELECT ON db.url_src +-- copy_of_function_src: +necessary to have the grant SELECT ON db.function_src +-- copy_of_url_src_no_password: +URL +-- copy_of_function_src_no_password: +Proxy +-- null_copy_of_url_src: +Null +after GRANT SELECT: +-- copy_of_url_src: +URL +-- copy_of_function_src: +Proxy +without SHOW COLUMNS: +-- local_src: +necessary to have the grant SHOW COLUMNS ON db.local_src +-- url_src: +necessary to have the grant SHOW COLUMNS ON db.url_src diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh new file mode 100755 index 0000000000000..22f7199b241af --- /dev/null +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# CREATE TABLE x AS y copies the engine of y with its credentials, which SHOW CREATE TABLE masks. +# That copy needs SELECT on y. Any other definition needs only SHOW COLUMNS. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +blind="blind_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +# the server does not contact the URLs. it only copies the definitions +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}, ${blind}; + CREATE USER ${user}, ${blind}; + GRANT CREATE TABLE ON ${db}.* TO ${user}, ${blind}; + GRANT TABLE ENGINE ON MergeTree, TABLE ENGINE ON Null TO ${user}, ${blind}; + GRANT URL ON *.* TO ${user}, ${blind}; + + CREATE TABLE ${db}.local_src (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE ${db}.url_src (id UInt64) ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.url_src_no_password (id UInt64) ENGINE = URL('http://127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.function_src (id UInt64) AS url('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.function_src_no_password (id UInt64) AS url('http://127.0.0.1:1/', 'CSV'); + + GRANT SHOW COLUMNS ON ${db}.local_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.url_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.url_src_no_password TO ${user}; + GRANT SHOW COLUMNS ON ${db}.function_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.function_src_no_password TO ${user}; +" + +# prints the missing privilege, or the engine of the copy. more arguments go to the client +function try_copy() +{ + local name=$1 source=$2 + shift 2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} --user "${user}" "${@}" -q "CREATE TABLE ${db}.${name} AS ${db}.${source}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_local_src local_src +try_copy copy_of_url_src url_src +try_copy copy_of_function_src function_src +# these two hold nothing masked, so the server copies them as before +try_copy copy_of_url_src_no_password url_src_no_password +try_copy copy_of_function_src_no_password function_src_no_password +# Null replaces the engine here, so the copy inherits nothing masked +try_copy null_copy_of_url_src url_src --restore_replace_external_engines_to_null 1 + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q " + GRANT SELECT ON ${db}.url_src TO ${user}; + GRANT SELECT ON ${db}.function_src TO ${user}; +" +try_copy copy_of_url_src url_src +try_copy copy_of_function_src function_src + +# a user who cannot see the source gets the same error for both tables +echo "without SHOW COLUMNS:" +for src in local_src url_src +do + echo "-- ${src}:" + ${CLICKHOUSE_CLIENT} --user "${blind}" -q "CREATE TABLE ${db}.blind_copy AS ${db}.${src}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" +done + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${blind}" diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference new file mode 100644 index 0000000000000..5a636ec83f4c1 --- /dev/null +++ b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference @@ -0,0 +1,6 @@ +with SHOW COLUMNS only: +necessary to have the grant SELECT ON db.kafka_src +with a password in this query: +Kafka +after GRANT SELECT: +Kafka diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh new file mode 100755 index 0000000000000..e810ad7954ede --- /dev/null +++ b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test does not build the Kafka engine + +# the engine settings can hold credentials, not only the engine arguments. SHOW CREATE TABLE masks +# kafka_sasl_password too, so a copy of it also needs SELECT. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +# the server does not contact the broker. it only copies the definition +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT KAFKA ON *.* TO ${user}; + + CREATE TABLE ${db}.kafka_src (id UInt64) ENGINE = Kafka + SETTINGS kafka_broker_list = 'localhost:9092', kafka_topic_list = 'topic', + kafka_group_name = 'group', kafka_format = 'CSV', kafka_sasl_password = 'secret'; + + GRANT SHOW COLUMNS ON ${db}.kafka_src TO ${user}; +" + +echo "with SHOW COLUMNS only:" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + +# the copy does not inherit a setting that this query gives, so it needs nothing more +echo "with a password in this query:" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.own_password AS ${db}.kafka_src SETTINGS kafka_sasl_password = 'own'" +${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'own_password'" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.kafka_src TO ${user}" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" +${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'copy_of_kafka_src'" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference new file mode 100644 index 0000000000000..eae6826a3c065 --- /dev/null +++ b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference @@ -0,0 +1,17 @@ +with SHOW COLUMNS only: +-- copy_legacy: +necessary to have the grant SELECT ON db.url_src +-- copy_current: +necessary to have the grant SELECT ON db.url_src +-- copy_legacy_unqualified: +necessary to have the grant SELECT ON db.url_src +-- copy_legacy_plain: +MergeTree +after GRANT SELECT: +-- copy_legacy: +URL +-- copy_legacy_unqualified: +URL +with SELECT but without the grant for the engine: +-- copy_no_url: +necessary to have the grant TABLE ENGINE ON URL diff --git a/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh new file mode 100755 index 0000000000000..a398557e009fb --- /dev/null +++ b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database +# Tag no-replicated-database: a Replicated database does not allow ON CLUSTER + +# an entry format before NORMALIZE_CREATE_ON_INITIATOR_VERSION holds the query as written, and the +# worker builds AS src with no user. the initiator must check the source itself. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +no_url="no_url_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT CLUSTER ON *.* TO ${user}; + GRANT URL ON *.* TO ${user}; + GRANT TABLE ENGINE ON MergeTree TO ${user}; + + DROP USER IF EXISTS ${no_url}; + CREATE USER ${no_url}; + GRANT CREATE TABLE ON ${db}.* TO ${no_url}; + GRANT CLUSTER ON *.* TO ${no_url}; + + CREATE TABLE ${db}.url_src (id UInt64) ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.plain_src (id UInt64) ENGINE = MergeTree ORDER BY id; + + GRANT SHOW COLUMNS ON ${db}.url_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.plain_src TO ${user}; + GRANT SELECT ON ${db}.url_src TO ${no_url}; +" + +legacy=(--user "${user}" --distributed_ddl_output_mode throw --distributed_ddl_entry_format_version 2) +current=(--user "${user}" --distributed_ddl_output_mode throw) + +# prints the missing privilege, or the engine of the copy. each case uses its own name +function try_copy() +{ + local name=$1 source=$2 + shift 2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} "${@}" -q "CREATE TABLE ${db}.${name} ON CLUSTER test_shard_localhost AS ${source}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ([A-Za-z]+|${db}\.[a-z_]+)" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_legacy "${db}.url_src" "${legacy[@]}" +try_copy copy_current "${db}.url_src" "${current[@]}" +# the server checks and reads a source without a database name in the database of this query +try_copy copy_legacy_unqualified url_src "${legacy[@]}" +# this one holds nothing masked, so the copy needs no SELECT +try_copy copy_legacy_plain "${db}.plain_src" "${legacy[@]}" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.url_src TO ${user}" +try_copy copy_legacy "${db}.url_src" "${legacy[@]}" +try_copy copy_legacy_unqualified url_src "${legacy[@]}" + +# the engine comes with the source, so the copy needs the grant for the engine too +echo "with SELECT but without the grant for the engine:" +try_copy copy_no_url "${db}.url_src" --user "${no_url}" --distributed_ddl_output_mode throw \ + --distributed_ddl_entry_format_version 2 + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${no_url}" diff --git a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference new file mode 100644 index 0000000000000..a24472ca367f7 --- /dev/null +++ b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference @@ -0,0 +1,6 @@ +with SHOW COLUMNS only: +-- copy_of_ts_src: +necessary to have the grant SELECT ON db.ts_src +after GRANT SELECT: +-- copy_of_ts_src: +TimeSeries diff --git a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh new file mode 100755 index 0000000000000..e110640472135 --- /dev/null +++ b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# a TimeSeries source holds its credentials in the inner engines, which the outer engine does not +# show. the copy inherits them, so it needs SELECT on the source. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" +client=(${CLICKHOUSE_CLIENT} --allow_experimental_time_series_table 1) + +# the URL is never contacted. only the definition is copied +"${client[@]}" -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT URL ON *.* TO ${user}; + GRANT TABLE ENGINE ON TimeSeries, TABLE ENGINE ON MergeTree, TABLE ENGINE ON AggregatingMergeTree, + TABLE ENGINE ON ReplacingMergeTree, TABLE ENGINE ON URL TO ${user}; + + CREATE TABLE ${db}.ts_src ENGINE = TimeSeries + SAMPLES ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + + GRANT SHOW COLUMNS ON ${db}.ts_src TO ${user}; +" + +# prints the missing privilege, or the engine of the copy +function try_copy() +{ + echo "-- ${1}:" + "${client[@]}" --user "${user}" -q "CREATE TABLE ${db}.${1} AS ${db}.ts_src" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${1}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_ts_src + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.ts_src TO ${user}" +try_copy copy_of_ts_src + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference new file mode 100644 index 0000000000000..e4ba420be9779 --- /dev/null +++ b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference @@ -0,0 +1,8 @@ +with SHOW COLUMNS only: +-- copy_of_key_src: +necessary to have the grant SELECT ON db.key_src +-- own_key: +s +after GRANT SELECT: +-- copy_of_key_src: +encrypt(\'aes-128-ecb\', \'[HIDDEN]\') diff --git a/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh new file mode 100755 index 0000000000000..8deb698cdb9c9 --- /dev/null +++ b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test does not build the encryption functions + +# a key expression can hold credentials too, and SHOW CREATE TABLE masks them. the copy needs SELECT +# on the source, unless this query gives its own key. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT TABLE ENGINE ON MergeTree TO ${user}; + + CREATE TABLE ${db}.key_src (s String) ENGINE = MergeTree ORDER BY encrypt('aes-128-ecb', s, '0123456789abcdef'); + + GRANT SHOW COLUMNS ON ${db}.key_src TO ${user}; +" + +# prints the missing privilege, or the sorting key of the copy +function try_copy() +{ + local name=$1 clauses=$2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.${name} AS ${db}.key_src ${clauses}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT sorting_key FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_key_src "" +try_copy own_key "ORDER BY s" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.key_src TO ${user}" +try_copy copy_of_key_src "" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference new file mode 100644 index 0000000000000..88c388e3e7f0d --- /dev/null +++ b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference @@ -0,0 +1,2 @@ +without SHOW COLUMNS: +necessary to have the grant SHOW COLUMNS ON db.plain_src diff --git a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh new file mode 100755 index 0000000000000..dc0aae0446d62 --- /dev/null +++ b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Tags: no-replicated-database +# Tag no-replicated-database: a Replicated database does not allow ON CLUSTER + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT CLUSTER ON *.* TO ${user}; + GRANT TABLE ENGINE ON MergeTree TO ${user}; + CREATE TABLE ${db}.plain_src (id UInt64) ENGINE = MergeTree ORDER BY id; +" + +echo "without SHOW COLUMNS:" +${CLICKHOUSE_CLIENT} --user "${user}" --distributed_ddl_output_mode throw \ + --distributed_ddl_entry_format_version 2 \ + -q "CREATE TABLE ${db}.copy ON CLUSTER test_shard_localhost ENGINE = MergeTree ORDER BY id AS ${db}.plain_src" 2>&1 \ + | grep -oE "necessary to have the grant SHOW COLUMNS ON ${db}\.plain_src" | sed "s/${db}/db/" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" From 07329845c409cc190d014aff46d85b64a92b4675 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 16:23:48 +0000 Subject: [PATCH 36/81] Update autogenerated version to 26.8.30.8 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- .../System/StorageSystemContributors.generated.cpp | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 09e487cd75e42..1729c22bbb464 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54542) +SET(VERSION_REVISION 54543) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 30) -SET(VERSION_GITHASH 7b8b36217c67c466bd4f903c15430b04e467b25a) -SET(VERSION_DESCRIBE v26.8.30.1-lts) -SET(VERSION_STRING 26.8.30.1) +SET(VERSION_PATCH 31) +SET(VERSION_GITHASH da45179b6c6761981c969859d168dd93b2bfd550) +SET(VERSION_DESCRIBE v26.8.31.1-lts) +SET(VERSION_STRING 26.8.31.1) # end of autochange diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 4c37b1b7ee935..fa3cb25e8b6af 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -1779,6 +1779,7 @@ const char * auto_contributors[] { "Yatian Xu", "Yatsishin Ilya", "Yağızcan Değirmenci", + "Ye Yuan", "Yecine Megdiche", "Yegor Andreenko", "Yegor Levankov", From 9882c7f241a071ce9cf0ad46b4f7ae44f3056510 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 18 Oct 2026 09:03:12 +0000 Subject: [PATCH 37/81] Backport #125467 to 26.8: Estimate an equality with a `Bool` literal at 50% when the compared expression has no statistics --- .../ConditionSelectivityEstimator.cpp | 41 ++++++++++++++----- .../ConditionSelectivityEstimator.h | 8 +++- .../join_swap_bool_literal_json_path.xml | 11 +++++ ...join_swap_bool_literal_json_path.reference | 13 ++++++ ...05366_join_swap_bool_literal_json_path.sql | 36 ++++++++++++++++ 5 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 tests/performance/join_swap_bool_literal_json_path.xml create mode 100644 tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.reference create mode 100644 tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.sql diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp index ec29fbfaada78..0442d50574c4d 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp @@ -360,6 +360,7 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr chassert(is_in_operator); const bool negative = func_name != "in"; const auto lhs_name = func.getArgumentAt(0).getColumnName(); + const DataTypePtr & set_type = prepared_set->getElementsTypes().front(); /// An expression rather than a column (`lower(col) IN (...)`) has no statistics to /// consult, and below the limit it is given a flat default by the "not a real column" @@ -367,9 +368,12 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr /// change the estimate for such an atom - and skip the size-based path, which would /// otherwise read `set_size / ` as "matches everything". if (metadata && !metadata->getColumns().tryGet(lhs_name)) - out.selectivity.true_sel = negative ? 1.0 - default_cond_equal_factor : default_cond_equal_factor; + { + const Float64 equal_factor = getEqualFactor(set_type, columns[0]->size()); + out.selectivity.true_sel = negative ? 1.0 - equal_factor : equal_factor; + } else - out.selectivity = estimateSelectivityFromSetSize(metadata, lhs_name, *columns[0], negative); + out.selectivity = estimateSelectivityFromSetSize(metadata, lhs_name, *columns[0], set_type, negative); out.finalized = true; return false; @@ -422,10 +426,14 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr /// Not a real column (e.g. a function expression like lower(col) or toDecimal64(col, 3)). /// Skip range analysis to avoid bad cast when merging ranges of incompatible Field types. /// Pre-set selectivity based on the function type so prewhere ordering is still reasonable. + /// Compared with a `Bool` literal or set, the expression is assumed to be boolean. + const Float64 equal_factor = set_for_ranges + ? getEqualFactor(set_for_ranges->getElementsTypes().front(), set_for_ranges->getTotalRowCount()) + : getEqualFactor(const_type); if (func_name == "equals" || func_name == "in") - out.selectivity.true_sel = default_cond_equal_factor; + out.selectivity.true_sel = equal_factor; else if (func_name == "notEquals" || func_name == "notIn") - out.selectivity.true_sel = 1.0 - default_cond_equal_factor; + out.selectivity.true_sel = 1.0 - equal_factor; else /// less, greater, lessOrEquals, greaterOrEquals out.selectivity.true_sel = default_cond_range_factor; out.finalized = true; @@ -622,7 +630,7 @@ ConditionSelectivityEstimator::Selectivity ConditionSelectivityEstimator::Column if (auto estimate = stats->estimateRange(range)) result += *estimate; else if (range.left == range.right) - result += static_cast(stats->getNonNullRowCount()) * default_cond_equal_factor; + result += static_cast(stats->getNonNullRowCount()) * getEqualFactor(stats->getDataType()); else result += static_cast(stats->getNonNullRowCount()) * default_cond_range_factor; } @@ -643,9 +651,18 @@ UInt64 ConditionSelectivityEstimator::ColumnEstimator::estimateCardinality() con return stats->estimateCardinality(); } +Float64 ConditionSelectivityEstimator::getEqualFactor(const DataTypePtr & type, size_t num_values) +{ + /// Like PostgreSQL, assume the two values of a `Bool` are equally frequent. + if (type && isBool(removeLowCardinalityAndNullable(type))) + return std::min(default_bool_equal_factor * static_cast(num_values), 1.0); + return default_cond_equal_factor; +} + ConditionSelectivityEstimator::Selectivity ConditionSelectivityEstimator::estimateSelectivityFromSetSize( - const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, bool negative) const + const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, + const DataTypePtr & set_type, bool negative) const { ProfileEvents::increment(ProfileEvents::SelectivityEstimatorInSetEstimatedFromSize); @@ -657,7 +674,7 @@ ConditionSelectivityEstimator::Selectivity ConditionSelectivityEstimator::estima if (it == column_estimators.end() || !isCompatibleStatistics(metadata, it->second.stats, column_name)) { /// No statistics: match what `finalize` assumes for a list of point ranges on an unknown column. - const Selectivity selectivity{std::min(static_cast(set_size) * default_cond_equal_factor, 1.0), 0}; + const Selectivity selectivity{std::min(static_cast(set_size) * getEqualFactor(set_type), 1.0), 0}; return negative ? selectivity.applyNot() : selectivity; } @@ -858,8 +875,10 @@ void ConditionSelectivityEstimator::RPNElement::finalize(const ColumnEstimators return; } - auto estimate_unknown_ranges = [&](const PlainRanges & ranges) -> Selectivity + auto estimate_unknown_ranges = [&](const String & column_name, const PlainRanges & ranges) -> Selectivity { + const ColumnDescription * column_desc = metadata ? metadata->getColumns().tryGet(column_name) : nullptr; + const Float64 equal_factor = getEqualFactor(column_desc ? column_desc->type : nullptr); Float64 equal_selectivity = 0; for (const Range & range : ranges.ranges) { @@ -867,7 +886,7 @@ void ConditionSelectivityEstimator::RPNElement::finalize(const ColumnEstimators return Selectivity{1.0, 0.0}; if (range.left == range.right) - equal_selectivity += default_cond_equal_factor; + equal_selectivity += equal_factor; else return Selectivity{default_cond_range_factor, 0}; } @@ -891,7 +910,7 @@ void ConditionSelectivityEstimator::RPNElement::finalize(const ColumnEstimators if (const auto * est = get_estimator(column_name)) estimate_results.emplace(column_name, est->estimateRanges(ranges)); else - estimate_results.emplace(column_name, estimate_unknown_ranges(ranges)); + estimate_results.emplace(column_name, estimate_unknown_ranges(column_name, ranges)); } for (const auto & [column_name, ranges] : column_not_ranges) @@ -900,7 +919,7 @@ void ConditionSelectivityEstimator::RPNElement::finalize(const ColumnEstimators if (const auto * est = get_estimator(column_name)) not_ranges_selectivity = est->estimateRanges(ranges).applyNot(); else - not_ranges_selectivity = estimate_unknown_ranges(ranges).applyNot(); + not_ranges_selectivity = estimate_unknown_ranges(column_name, ranges).applyNot(); auto it = estimate_results.find(column_name); if (it == estimate_results.end()) diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h index f96669320dd1c..f198ef66c0ae9 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.h +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h @@ -130,12 +130,18 @@ class ConditionSelectivityEstimator : public WithContext /// can possibly cover. Costs one pass for the bounds and a single statistics probe, where turning the /// set into ranges costs a `Field` per element, a sort and one probe per element. Selectivity estimateSelectivityFromSetSize( - const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, bool negative) const; + const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, + const DataTypePtr & set_type, bool negative) const; UInt64 estimateSelectivity(const RPNBuilderTreeNode & node) const; + /// Selectivity of an equality with any of `num_values` values of `type` when no statistics can estimate it. + /// Only a `Bool` counts the values: the default for other types is flat. + static Float64 getEqualFactor(const DataTypePtr & type, size_t num_values = 1); + /// Magic constants for estimating the selectivity of a condition no statistics exists. static constexpr Float64 default_cond_range_factor = 0.33; static constexpr Float64 default_cond_equal_factor = 0.01; + static constexpr Float64 default_bool_equal_factor = 0.5; static constexpr Float64 default_unknown_cond_factor = 0.33; static constexpr Float64 default_like_factor = 0.1; static constexpr Float64 default_cardinality_ratio = 0.1; diff --git a/tests/performance/join_swap_bool_literal_json_path.xml b/tests/performance/join_swap_bool_literal_json_path.xml new file mode 100644 index 0000000000000..1d93003e38dd9 --- /dev/null +++ b/tests/performance/join_swap_bool_literal_json_path.xml @@ -0,0 +1,11 @@ + + CREATE TABLE join_bool_small (k UInt64) ENGINE = MergeTree ORDER BY k + CREATE TABLE join_bool_json (k UInt64, j JSON) ENGINE = MergeTree ORDER BY k + INSERT INTO join_bool_small SELECT number FROM numbers_mt(1000000) + INSERT INTO join_bool_json SELECT number, if(number % 10 = 0, '{{"f":true}}', '{{"f":false}}') FROM numbers_mt(10000000) + ALTER TABLE join_bool_small MATERIALIZE STATISTICS ALL SETTINGS mutations_sync = 1 + ALTER TABLE join_bool_json MATERIALIZE STATISTICS ALL SETTINGS mutations_sync = 1 + SELECT count() FROM join_bool_small AS s LEFT JOIN join_bool_json AS t ON s.k = t.k WHERE t.j.f = false FORMAT Null + DROP TABLE IF EXISTS join_bool_small + DROP TABLE IF EXISTS join_bool_json + diff --git a/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.reference b/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.reference new file mode 100644 index 0000000000000..c2ba0791c261f --- /dev/null +++ b/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.reference @@ -0,0 +1,13 @@ +t2 +t1 +t2 +t1 +5000 +5000 +5000 +5000 +10000 +10000 +10000 +900 +100 diff --git a/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.sql b/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.sql new file mode 100644 index 0000000000000..3facf6d7c1932 --- /dev/null +++ b/tests/queries/0_stateless/05366_join_swap_bool_literal_json_path.sql @@ -0,0 +1,36 @@ +-- Tags: no-parallel-replicas +-- Random settings limits: query_plan_optimize_join_order_randomize=(0, 0); query_plan_optimize_join_order_limit=(1, None); use_statistics=(1, None); materialize_statistics_on_insert=(1, None); query_plan_convert_outer_join_to_inner_join=(1, None) + +-- A comparison of a JSON path with a Bool literal is estimated at half of the rows, so the join builds the smaller side. + +SET query_plan_join_swap_table = 'auto'; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +CREATE TABLE t1 (k UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (k UInt64, j JSON) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t3 (k UInt64 STATISTICS(uniq), a Bool, b Bool STATISTICS(tdigest)) ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; +INSERT INTO t1 SELECT number FROM numbers(1000); +INSERT INTO t2 SELECT number, if(number % 10 = 0, '{"f":true}', '{"f":false}') FROM numbers(10000); +INSERT INTO t3 SELECT number, number % 10 = 0, number % 10 = 0 FROM numbers(10000); + +SELECT if(explain ILIKE '%.t1)%', 't1', 't2') FROM (EXPLAIN SELECT count() FROM t1 LEFT JOIN t2 ON t1.k = t2.k WHERE t2.j.f = false) WHERE explain ILIKE '%ReadFromMergeTree%'; +SELECT if(explain ILIKE '%.t1)%', 't1', 't2') FROM (EXPLAIN SELECT count() FROM t1 LEFT JOIN t2 ON t1.k = t2.k WHERE t2.j.f = true) WHERE explain ILIKE '%ReadFromMergeTree%'; +SELECT extract(explain, 't2\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t2 ON t1.k = t2.k WHERE t2.j.f = false) WHERE explain LIKE '%Join:%' LIMIT 1; +SELECT extract(explain, 't2\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t2 ON t1.k = t2.k WHERE t2.j.f != false) WHERE explain LIKE '%Join:%' LIMIT 1; + +-- The same for a `Bool` column without statistics, a `Bool` column whose statistics cannot estimate an equality, +-- and a `Bool` expression in a set, each also above the set size limit. +SELECT extract(explain, 't3\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t3 ON t1.k = t3.k WHERE t3.a = true) WHERE explain LIKE '%Join:%' LIMIT 1; +SELECT extract(explain, 't3\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t3 ON t1.k = t3.k WHERE t3.b = true) WHERE explain LIKE '%Join:%' LIMIT 1; +SELECT extract(explain, 't3\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t3 ON t1.k = t3.k WHERE t3.a IN (false, true) SETTINGS statistics_max_set_size_for_exact_selectivity_estimation = 1) WHERE explain LIKE '%Join:%' LIMIT 1; +SELECT extract(explain, 't3\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t3 ON t1.k = t3.k WHERE toBool(t3.k % 2) IN (false, true)) WHERE explain LIKE '%Join:%' LIMIT 1; +SELECT extract(explain, 't3\\[(\\d+)\\]') FROM (EXPLAIN PLAN keep_logical_steps = 1 SELECT count() FROM t1 INNER JOIN t3 ON t1.k = t3.k WHERE toBool(t3.k % 2) IN (false, true) SETTINGS statistics_max_set_size_for_exact_selectivity_estimation = 1) WHERE explain LIKE '%Join:%' LIMIT 1; + +SELECT count() FROM t1 LEFT JOIN t2 ON t1.k = t2.k WHERE t2.j.f = false; +SELECT count() FROM t1 LEFT JOIN t2 ON t1.k = t2.k WHERE t2.j.f = true; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; From b2f3dde735d037032c3d2e0ca69cabe8d89a713e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 18 Oct 2026 21:32:47 +0000 Subject: [PATCH 38/81] Backport #126120 to 26.8: Fix out-of-bounds accesses in the replxx line editor --- contrib/replxx | 2 +- .../05401_client_history_search_crash.expect | 91 +++++++++++++++++++ ...5401_client_history_search_crash.reference | 0 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100755 tests/queries/0_stateless/05401_client_history_search_crash.expect create mode 100644 tests/queries/0_stateless/05401_client_history_search_crash.reference diff --git a/contrib/replxx b/contrib/replxx index 00cd994b363c3..191e04e43b1ed 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 00cd994b363c3fc32f80d9d266e4d2acb4159abf +Subproject commit 191e04e43b1ed72289adfe7d2b3fd59b4fb4e78b diff --git a/tests/queries/0_stateless/05401_client_history_search_crash.expect b/tests/queries/0_stateless/05401_client_history_search_crash.expect new file mode 100755 index 0000000000000..48852f89ae896 --- /dev/null +++ b/tests/queries/0_stateless/05401_client_history_search_crash.expect @@ -0,0 +1,91 @@ +#!/usr/bin/expect -f + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +if {[info exists env(CLICKHOUSE_TMP)]} { + set CLICKHOUSE_TMP $env(CLICKHOUSE_TMP) +} else { + set CLICKHOUSE_TMP "." +} +exp_internal -f $CLICKHOUSE_TMP/$basename.debuglog 0 +set history_file $CLICKHOUSE_TMP/$basename.history + +log_user 0 +set timeout 60 +match_max 100000 + +expect_after { + # Do not ignore eof from expect + -i $any_spawn_id eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + -i $any_spawn_id timeout { exit 1 } +} + +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file" +expect ":) " + +send -- "SELECT 1\r" +expect "1" +expect ":) " + +# Recall the query (Ctrl-P) and make it longer than the history entry. +send -- "\x10" +expect "SELECT 1" +send -- "23" +expect "SELECT 123" + +# A forward search (Ctrl-S) that finds nothing, left with Ctrl-F, must keep the edited line and the cursor in it. +send -- "\x13" +expect "(i-search)" +send -- "\"" +expect "(i-search)`\"'" +send -- "\x06" +expect ":) " +send -- "+4\r" +expect -re {\s127\s} +expect ":) " + +# Paste a query with a tab, then start a search seeded with the line (Meta-R), which copies the tab into the search prompt. +send -- "\033\[200~SELECT\t40 + 2\033\[201~" +expect "SELECT 40 + 2" +send -- "\033r" +expect "(reverse-i-search)" +send -- "\x07" +expect ":) " +send -- "\r" +expect -re {\s42\s} +expect ":) " + +send -- "\4" +expect eof + +# Leave a history search with a prompt that contains a double-width character. +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file --prompt '日 :) '" +expect "日 :) " +send -- "\x14" +expect "(reverse-i-search)" +send -- "x" +send -- "\x07" +expect "日 :) " +send -- "SELECT 50 - 8\r" +expect -re {\s42\s} +expect "日 :) " +send -- "\4" +expect eof + +# Ctrl-W and Alt-. must keep U+10FFFF inside a word, sent as raw UTF-8 bytes (Tcl 8.6 needs a binary channel for that). +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file" +fconfigure $spawn_id -encoding binary +expect ":) " +send -- "SELECT 'a\xf4\x8f\xbf\xbfz" +send -- "\x17" +send -- "41 + 1 AS `b\xf4\x8f\xbf\xbf`\r" +expect -re {\s42\s} +expect ":) " +send -- "SELECT 6 * 7 AS " +send -- "\033." +send -- "\r" +expect -re {\s42\s} +expect ":) " +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/05401_client_history_search_crash.reference b/tests/queries/0_stateless/05401_client_history_search_crash.reference new file mode 100644 index 0000000000000..e69de29bb2d1d From 5cc9182ba704add9f6b0773de85d4034256daaf5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 09:28:10 +0000 Subject: [PATCH 39/81] Backport #123195 to 26.8: Retry S3 backup and restore requests after ExpiredToken --- src/Backups/BackupIO_S3.cpp | 5 + src/Core/Settings.cpp | 3 + src/Core/SettingsChangesHistory.cpp | 1 + src/IO/S3/Client.cpp | 42 ++- src/IO/S3/PocoHTTPClient.h | 1 + .../__init__.py | 0 .../configs/s3.xml | 26 ++ .../s3_mocks/mock_sts.py | 56 ++++ .../s3_mocks/s3_proxy.py | 178 +++++++++++ .../test.py | 292 ++++++++++++++++++ ...xpired_token_retry_compatibility.reference | 2 + ...p_s3_expired_token_retry_compatibility.sql | 2 + 12 files changed, 605 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_backup_restore_s3_expired_token/__init__.py create mode 100644 tests/integration/test_backup_restore_s3_expired_token/configs/s3.xml create mode 100644 tests/integration/test_backup_restore_s3_expired_token/s3_mocks/mock_sts.py create mode 100644 tests/integration/test_backup_restore_s3_expired_token/s3_mocks/s3_proxy.py create mode 100644 tests/integration/test_backup_restore_s3_expired_token/test.py create mode 100644 tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.reference create mode 100644 tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.sql diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index f6875f0830474..9af9b26524782 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -35,6 +35,7 @@ namespace DB namespace Setting { extern const SettingsUInt64 backup_restore_s3_retry_attempts; + extern const SettingsUInt64 backup_s3_expired_token_retry_timeout_ms; extern const SettingsUInt64 backup_restore_s3_retry_initial_backoff_ms; extern const SettingsUInt64 backup_restore_s3_retry_max_backoff_ms; extern const SettingsFloat backup_restore_s3_retry_jitter_factor; @@ -102,6 +103,7 @@ class S3BackupClientCreator .max_delay_ms = static_cast(local_settings[Setting::backup_restore_s3_retry_max_backoff_ms]), .jitter_factor = static_cast(local_settings[Setting::backup_restore_s3_retry_jitter_factor])}; slow_all_threads_after_retryable_error = local_settings[Setting::backup_slow_all_threads_after_retryable_s3_error]; + expired_token_retry_timeout_ms = local_settings[Setting::backup_s3_expired_token_retry_timeout_ms]; } S3BackupDiskClientFactory::Entry operator()(DiskPtr disk) const @@ -111,6 +113,7 @@ class S3BackupClientCreator auto config = disk_client->getClientConfiguration(); config.retry_strategy = retry_strategy; config.s3_slow_all_threads_after_retryable_error = slow_all_threads_after_retryable_error; + config.expired_token_retry_timeout_ms = expired_token_retry_timeout_ms; return {disk_client->cloneWithConfigurationOverride(config), disk_client}; } @@ -118,6 +121,7 @@ class S3BackupClientCreator private: S3::PocoHTTPClientConfiguration::RetryStrategy retry_strategy; bool slow_all_threads_after_retryable_error = false; + UInt64 expired_token_retry_timeout_ms = 0; }; std::shared_ptr makeS3Client( @@ -222,6 +226,7 @@ class S3BackupClientCreator s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; + client_configuration.expired_token_retry_timeout_ms = local_settings[Setting::backup_s3_expired_token_retry_timeout_ms]; client_configuration.maxConnections = static_cast(global_settings[Setting::s3_max_connections]); /// Increase connect timeout client_configuration.connectTimeoutMs = 10 * 1000; diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index b27e2e54369ac..ac09d124a1734 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -4154,6 +4154,9 @@ Approximate probability of failure for a keeper request during backup or restore )", 0) \ DECLARE(UInt64, backup_restore_s3_retry_attempts, 1000, R"( Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore. +)", 0) \ + DECLARE(UInt64, backup_s3_expired_token_retry_timeout_ms, 120000, R"( +Maximum time in milliseconds to retry an S3 request whose response identifies `ExpiredToken` during backup or restore. Retries also stop at `backup_restore_s3_retry_attempts`. Zero disables these retries. AWS S3 `HEAD` responses expose only a generic HTTP status, so this retry does not cover `HEAD` metadata requests. )", 0) \ DECLARE(UInt64, backup_restore_s3_retry_initial_backoff_ms, 25, R"( Initial backoff delay in milliseconds before the first retry attempt during backup and restore. Each subsequent retry increases the delay exponentially, up to the maximum specified by `backup_restore_s3_retry_max_backoff_ms` diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a54bbc97a1b7e..d272210c67776 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -47,6 +47,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8", { + {"backup_s3_expired_token_retry_timeout_ms", 0, 120000, "Retry S3 backup and restore requests after an expired token; 0 restores the previous behavior."}, {"allow_named_collection_override_by_default", true, true, "Obsolete. Overriding named collection keys requires `SHOW NAMED COLLECTIONS SECRETS`."}, {"use_aggregation_memory_tracker", false, false, "New setting. In this release it is disabled by default, which restores the two-level conversion of `GROUP BY` from before 26.7 (https://github.com/ClickHouse/ClickHouse/pull/107490)."}, {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.8 restores the previous strict behavior."}, diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 874a1509c60fa..dc9acdbdaa3e4 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -359,7 +360,7 @@ Aws::Auth::AWSCredentials Client::getCredentials() const bool Client::checkIfCredentialsChanged(const Aws::S3::S3Error & error) const { - return (error.GetExceptionName() == "AuthenticationRequired"); + return error.GetExceptionName() == "AuthenticationRequired" || error.GetExceptionName() == "ExpiredToken"; } bool Client::checkIfWrongRegionDefined(const std::string & bucket, const Aws::S3::S3Error & error, std::string & region) const @@ -704,8 +705,15 @@ Client::doRequest(RequestType & request, RequestFn request_fn) const } ); - for (size_t attempt = 0; attempt <= max_redirects; ++attempt) + std::optional expired_token_retry_watch; + size_t expired_token_retries = 0; + for (size_t attempt = 0; ; ++attempt) { + chassert(attempt >= expired_token_retries); + const size_t effective_redirect_attempt = attempt - expired_token_retries; + if (effective_redirect_attempt > max_redirects) + break; + auto result = request_fn(request); if (result.IsSuccess()) return result; @@ -716,8 +724,36 @@ Client::doRequest(RequestType & request, RequestFn request_fn) const if (checkIfCredentialsChanged(error)) { - LOG_INFO(log, "Credentials changed, attempting again"); + if (error.GetExceptionName() != "ExpiredToken") + { + LOG_INFO(log, "Credentials changed, attempting again"); + credentials_provider->SetNeedRefresh(); + continue; + } + + const auto timeout_ms = client_configuration.expired_token_retry_timeout_ms; + if (timeout_ms == 0) + return result; + + if (!expired_token_retry_watch) + expired_token_retry_watch.emplace(CLOCK_MONOTONIC); + + const auto elapsed_ms = expired_token_retry_watch->elapsedMilliseconds(); + if (expired_token_retries >= client_configuration.retry_strategy.max_retries || elapsed_ms >= timeout_ms) + return result; + + RetryStrategy retry_strategy(client_configuration.retry_strategy); + const auto retry_delay_ms = static_cast(retry_strategy.CalculateDelayBeforeNextRetry(error, expired_token_retries)); + if (retry_delay_ms >= timeout_ms - elapsed_ms) + return result; + + if (expired_token_retries == 0) + LOG_INFO(log, "ExpiredToken on S3 request, refreshing credentials and retrying"); + ++expired_token_retries; credentials_provider->SetNeedRefresh(); + sleepForMilliseconds(retry_delay_ms); + if (expired_token_retry_watch->elapsedMilliseconds() >= timeout_ms) + return result; continue; } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 0cda72dc47f2a..5932e0f8b820b 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -69,6 +69,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects = DEFAULT_MAX_REDIRECTS; RetryStrategy retry_strategy; + UInt64 expired_token_retry_timeout_ms = 0; bool s3_slow_all_threads_after_network_error; bool s3_slow_all_threads_after_retryable_error; bool enable_s3_requests_logging; diff --git a/tests/integration/test_backup_restore_s3_expired_token/__init__.py b/tests/integration/test_backup_restore_s3_expired_token/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_backup_restore_s3_expired_token/configs/s3.xml b/tests/integration/test_backup_restore_s3_expired_token/configs/s3.xml new file mode 100644 index 0000000000000..5c1ab8d088ea6 --- /dev/null +++ b/tests/integration/test_backup_restore_s3_expired_token/configs/s3.xml @@ -0,0 +1,26 @@ + + + 1 + + + + + s3 + http://s3-proxy:8080/root/expired_token_source/ + true + 1 + arn::role + expired-token-test + + + + + +
+ s3_source +
+ + + + + diff --git a/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/mock_sts.py b/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/mock_sts.py new file mode 100644 index 0000000000000..eb06cdf3fb09a --- /dev/null +++ b/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/mock_sts.py @@ -0,0 +1,56 @@ +import sys + +from bottle import request, response, route, run + +credential_sets = None +credential_index = 0 + + +@route("/") +def ping(): + return "OK" + + +@route("/toggle", method="POST") +def toggle(): + global credential_index + if credential_sets is None: + response.status = 503 + return "Credentials are not configured" + credential_index = 1 - credential_index + return "OK" + + +@route("/set", method="POST") +def set_credentials(): + global credential_sets, credential_index + configured = request.json["credentials"] + assert len(configured) == 2 + credential_sets = configured + credential_index = 0 + return "OK" + + +@route("/", method="POST") +def assume_role(): + assert request.query.get("RoleArn") == "arn::role" + if credential_sets is None: + response.status = 503 + return "Credentials are not configured" + credentials = credential_sets[credential_index] + response.content_type = "text/xml" + return f""" + + + + {credentials['AccessKeyId']} + {credentials['SecretAccessKey']} + {credentials['SessionToken']} + {credentials['Expiration']} + + + +""" + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/s3_proxy.py b/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/s3_proxy.py new file mode 100644 index 0000000000000..fe6e4e1807244 --- /dev/null +++ b/tests/integration/test_backup_restore_s3_expired_token/s3_mocks/s3_proxy.py @@ -0,0 +1,178 @@ +import http.client +import hashlib +import json +import re +import sys +import threading +import time +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +state_lock = threading.Lock() +armed = None +events = [] + + +def operation(method, path, headers): + query = urllib.parse.parse_qs(urllib.parse.urlsplit(path).query) + if method == "PUT" and "partNumber" in query and "x-amz-copy-source" in headers: + return "UploadPartCopy" + if method == "PUT" and "partNumber" in query: + return "UploadPart" + if method == "GET": + return "GetObject" + return None + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def reply(self, status, body=b"", headers=None): + self.send_response(status) + for name, value in (headers or {}).items(): + if name.lower() not in ( + "content-length", + "transfer-encoding", + "connection", + ): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def read_body(self): + if self.headers.get("Transfer-Encoding", "").lower() != "chunked": + return self.rfile.read(int(self.headers.get("Content-Length", 0))) + + chunks = [] + while True: + line = self.rfile.readline() + chunks.append(line) + size = int(line.split(b";", 1)[0], 16) + if size == 0: + while True: + trailer = self.rfile.readline() + chunks.append(trailer) + if trailer in (b"\r\n", b"\n", b""): + return b"".join(chunks) + chunks.append(self.rfile.read(size + 2)) + + def handle_control(self): + global armed + if self.path == "/control/events": + with state_lock: + events_snapshot = list(events) + self.reply( + 200, + json.dumps(events_snapshot).encode(), + {"Content-Type": "application/json"}, + ) + return + if self.path.startswith("/control/arm?"): + params = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query) + with state_lock: + events.clear() + armed = { + "operation": params["operation"][0], + "prefix": params["prefix"][0], + "remaining": int(params["count"][0]), + "rotate": params.get("rotate", ["0"])[0] == "1", + "delay_ms": int(params.get("delay_ms", ["0"])[0]), + } + self.reply(200, b"OK") + return + self.reply(404) + + def proxy(self): + global armed + if self.path == "/": + self.reply(200, b"OK") + return + if self.path.startswith("/control/"): + self.handle_control() + return + + body = self.read_body() + current_operation = operation(self.command, self.path, self.headers) + match = re.search(r"Credential=([^/]+)", self.headers.get("Authorization", "")) + access_key = match.group(1) if match else "" + token = self.headers.get("x-amz-security-token", "") + token_digest = hashlib.sha256(token.encode()).hexdigest() if token else "" + + inject = False + rotate = False + delay_ms = 0 + with state_lock: + if ( + armed + and current_operation == armed["operation"] + and self.path.startswith(armed["prefix"]) + ): + inject = armed["remaining"] > 0 + if inject: + armed["remaining"] -= 1 + rotate = armed["rotate"] + armed["rotate"] = False + delay_ms = armed["delay_ms"] + events.append( + { + "operation": current_operation, + "path": self.path, + "access_key": access_key, + "token_digest": token_digest, + "body_size": len(body), + "injected": inject, + } + ) + + if inject: + if delay_ms: + time.sleep(delay_ms / 1000) + if rotate: + urllib.request.urlopen( + urllib.request.Request( + "http://sts.amazonaws.com:80/toggle", method="POST" + ), + timeout=5, + ).close() + self.reply( + 400, + b"ExpiredTokenThe provided token has expired.", + {"Content-Type": "application/xml", "x-amz-request-id": "injected"}, + ) + return + + connection = http.client.HTTPConnection("minio1", 9001, timeout=30) + connection.putrequest( + self.command, self.path, skip_host=True, skip_accept_encoding=True + ) + for name, value in self.headers.items(): + if name.lower() != "connection": + connection.putheader(name, value) + connection.endheaders(body) + upstream = connection.getresponse() + result = upstream.read() + headers = dict(upstream.getheaders()) + if self.command == "HEAD": + self.send_response(upstream.status) + for name, value in headers.items(): + if name.lower() not in ("transfer-encoding", "connection"): + self.send_header(name, value) + self.end_headers() + else: + self.reply(upstream.status, result, headers) + connection.close() + + do_GET = proxy + do_HEAD = proxy + do_PUT = proxy + do_POST = proxy + do_DELETE = proxy + + +ThreadingHTTPServer(("0.0.0.0", int(sys.argv[1])), Handler).serve_forever() diff --git a/tests/integration/test_backup_restore_s3_expired_token/test.py b/tests/integration/test_backup_restore_s3_expired_token/test.py new file mode 100644 index 0000000000000..64d9350e9175b --- /dev/null +++ b/tests/integration/test_backup_restore_s3_expired_token/test.py @@ -0,0 +1,292 @@ +import hashlib +import logging +import os +import uuid + +import boto3 +import pytest +import requests + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key +from helpers.mock_servers import start_mock_servers + +ROLE = "extra_credentials(role_arn = 'arn::role', role_session_name = 'expired-token-test')" +BACKUP_SETTINGS = { + "backup_restore_s3_retry_attempts": 2, + "backup_restore_s3_retry_initial_backoff_ms": 10, + "backup_restore_s3_retry_max_backoff_ms": 10, + "backup_restore_s3_retry_jitter_factor": 0, + "s3_max_single_operation_copy_size": 5 * 1024 * 1024, +} + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + with_minio=True, + main_configs=["configs/s3.xml"], + env_variables={"AWS_ACCESS_KEY_ID": "aws", "AWS_SECRET_ACCESS_KEY": "aws123"}, + ) + sts = cluster.add_instance( + name="sts.amazonaws.com", + hostname="sts.amazonaws.com", + image="clickhouse/python-bottle", + tag="latest", + stay_alive=True, + ) + sts.stop_clickhouse(kill=True) + proxy = cluster.add_instance( + name="s3-proxy", + hostname="s3-proxy", + image="clickhouse/python-bottle", + tag="latest", + stay_alive=True, + ) + proxy.stop_clickhouse(kill=True) + + try: + cluster.start() + cluster.exec_in_container( + cluster.minio_docker_id, + [ + "mc", + "alias", + "set", + "root", + "http://minio1:9001", + "minio", + minio_secret_key, + ], + ) + cluster.exec_in_container( + cluster.minio_docker_id, + [ + "mc", + "admin", + "user", + "add", + "root", + "minio_rotated", + "Rotated_Minio_Test_Secret_123", + ], + ) + cluster.exec_in_container( + cluster.minio_docker_id, + [ + "mc", + "admin", + "policy", + "attach", + "root", + "readwrite", + "--user", + "minio_rotated", + ], + ) + + minio_url = f"http://{cluster.minio_ip}:{cluster.minio_port}" + botocore_logger = logging.getLogger("botocore") + previous_log_level = botocore_logger.level + botocore_logger.setLevel(logging.INFO) + try: + sts_client = boto3.client( + "sts", + endpoint_url=minio_url, + region_name="us-east-1", + aws_access_key_id="minio_rotated", + aws_secret_access_key="Rotated_Minio_Test_Secret_123", + ) + credential_sets = [] + for index in range(2): + issued = sts_client.assume_role( + RoleArn="arn:aws:iam::123456789012:role/expired-token-test", + RoleSessionName=f"expired-token-test-{index}", + DurationSeconds=3600, + )["Credentials"] + credentials = { + key: issued[key] + for key in ("AccessKeyId", "SecretAccessKey", "SessionToken") + } + credentials["Expiration"] = issued["Expiration"].strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + boto3.client( + "s3", + endpoint_url=minio_url, + region_name="us-east-1", + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], + ).head_bucket(Bucket="root") + credential_sets.append(credentials) + finally: + botocore_logger.setLevel(previous_log_level) + + if ( + not all(credentials["SessionToken"] for credentials in credential_sets) + or credential_sets[0]["AccessKeyId"] == credential_sets[1]["AccessKeyId"] + or credential_sets[0]["SessionToken"] == credential_sets[1]["SessionToken"] + ): + raise AssertionError( + "MinIO STS did not return two distinct credential sets" + ) + + script_dir = os.path.join(os.path.dirname(__file__), "s3_mocks") + start_mock_servers( + cluster, + script_dir, + [ + ("mock_sts.py", "sts.amazonaws.com", "80", []), + ("s3_proxy.py", "s3-proxy", "8080", []), + ], + ) + response = requests.post( + f"http://{cluster.get_instance_ip('sts.amazonaws.com')}:80/set", + json={"credentials": credential_sets}, + timeout=10, + ) + response.raise_for_status() + yield cluster, credential_sets + finally: + cluster.shutdown() + + +def proxy_url(cluster): + return f"http://{cluster.get_instance_ip('s3-proxy')}:8080" + + +def arm(cluster, operation, prefix, count=1, rotate=False, delay_ms=0): + response = requests.post( + f"{proxy_url(cluster)}/control/arm", + params={ + "operation": operation, + "prefix": prefix, + "count": count, + "rotate": int(rotate), + "delay_ms": delay_ms, + }, + timeout=10, + ) + response.raise_for_status() + + +def events(cluster): + response = requests.get(f"{proxy_url(cluster)}/control/events", timeout=10) + response.raise_for_status() + return response.json() + + +def backup_destination(path): + return f"S3('http://s3-proxy:8080/root/{path}', {ROLE})" + + +def backup(node, path, settings): + return node.query( + f"BACKUP TABLE expired_token_data TO {backup_destination(path)}", + settings=settings, + ) + + +def credential_identity(credentials): + return ( + credentials["AccessKeyId"], + hashlib.sha256(credentials["SessionToken"].encode()).hexdigest(), + ) + + +def matching_retry(events_, old_credentials, new_credentials): + injected_events = [event for event in events_ if event["injected"]] + assert injected_events, events_ + injected = injected_events[0] + assert (injected["access_key"], injected["token_digest"]) == credential_identity( + old_credentials + ), events_ + assert any( + event["path"] == injected["path"] + and not event["injected"] + and (event["access_key"], event["token_digest"]) + == credential_identity(new_credentials) + for event in events_ + ), events_ + + +def test_backup_restore_refreshes_expired_s3_credentials(started_cluster): + cluster, (credentials_a, credentials_b) = started_cluster + node = cluster.instances["node"] + node.query( + "CREATE TABLE expired_token_data (value String) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy = 's3_source'" + ) + node.query("INSERT INTO expired_token_data SELECT randomString(8 * 1024 * 1024)") + + path = f"backups/expired_token_{uuid.uuid4().hex}" + arm(cluster, "UploadPartCopy", f"/root/{path}/", rotate=True) + assert "BACKUP_CREATED" in backup(node, path, BACKUP_SETTINGS) + matching_retry(events(cluster), credentials_a, credentials_b) + + arm(cluster, "GetObject", f"/root/{path}/", rotate=True) + result = node.query( + f"RESTORE TABLE expired_token_data AS restored_data FROM {backup_destination(path)}", + settings=BACKUP_SETTINGS, + ) + assert "RESTORED" in result + matching_retry(events(cluster), credentials_b, credentials_a) + assert node.query("SELECT cityHash64(value) FROM restored_data") == node.query( + "SELECT cityHash64(value) FROM expired_token_data" + ) + + node.query( + "CREATE TABLE local_data (value String) ENGINE = MergeTree ORDER BY tuple()" + ) + node.query("INSERT INTO local_data SELECT randomString(8 * 1024 * 1024)") + upload_path = f"backups/expired_token_upload_{uuid.uuid4().hex}" + arm(cluster, "UploadPart", f"/root/{upload_path}/", rotate=True) + result = node.query( + f"BACKUP TABLE local_data TO {backup_destination(upload_path)}", + settings={**BACKUP_SETTINGS, "s3_max_single_part_upload_size": 0}, + ) + assert "BACKUP_CREATED" in result + upload_events = events(cluster) + matching_retry(upload_events, credentials_a, credentials_b) + assert next(event for event in upload_events if event["injected"])["body_size"] > 0 + assert "RESTORED" in node.query( + f"RESTORE TABLE local_data AS restored_local FROM {backup_destination(upload_path)}", + settings=BACKUP_SETTINGS, + ) + assert node.query("SELECT cityHash64(value) FROM restored_local") == node.query( + "SELECT cityHash64(value) FROM local_data" + ) + + for timeout_ms, attempts, delay_ms, expected_requests in [ + (5000, 2, 0, 3), + (200, 2, 500, 2), + (1, 2, 0, 1), + (0, 2, 0, 1), + ]: + failing_path = f"backups/expired_token_limit_{uuid.uuid4().hex}" + arm( + cluster, + "UploadPartCopy", + f"/root/{failing_path}/", + count=100, + delay_ms=delay_ms, + ) + error = node.query_and_get_error( + f"BACKUP TABLE expired_token_data TO {backup_destination(failing_path)}", + settings={ + **BACKUP_SETTINGS, + "backup_s3_expired_token_retry_timeout_ms": timeout_ms, + "backup_restore_s3_retry_attempts": attempts, + }, + ) + assert "ExpiredToken" in error, error + observed = events(cluster) + injected = [event for event in observed if event["injected"]] + assert injected, observed + first_path = injected[0]["path"] + assert ( + len([event for event in observed if event["path"] == first_path]) + == expected_requests + ), observed diff --git a/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.reference b/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.reference new file mode 100644 index 0000000000000..72c2027c0fc0a --- /dev/null +++ b/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.reference @@ -0,0 +1,2 @@ +0 +120000 diff --git a/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.sql b/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.sql new file mode 100644 index 0000000000000..342d8b6a02808 --- /dev/null +++ b/tests/queries/0_stateless/05399_backup_s3_expired_token_retry_compatibility.sql @@ -0,0 +1,2 @@ +SELECT getSetting('backup_s3_expired_token_retry_timeout_ms') SETTINGS compatibility = '26.7'; +SELECT getSetting('backup_s3_expired_token_retry_timeout_ms') SETTINGS compatibility = '26.8'; From 301559ff75b5c474c8765e568e4ea418df8cbf9c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 11:33:52 +0000 Subject: [PATCH 40/81] Backport #113877 to 26.8: Fix out-of-bounds write on a long codec chain --- src/Compression/CompressionCodecGorilla.cpp | 5 +- src/Compression/CompressionCodecMultiple.cpp | 62 +++++- .../CompressionFactoryAdditions.cpp | 58 ++++++ src/Storages/AlterCommands.cpp | 4 +- ...812_codec_chain_reserve_overflow.reference | 14 ++ .../04812_codec_chain_reserve_overflow.sql | 176 ++++++++++++++++++ 6 files changed, 311 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference create mode 100644 tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 083d2abedb3dc..8c937ac5b4875 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -167,9 +167,10 @@ UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { const UInt32 items_count = uncompressed_size / data_bytes_size; - static const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); + /// Not static: data_bytes_size is an argument, so a static keeps the first caller's width. + const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); // -1 since there must be at least 1 non-zero bit. - static const auto LEADING_ZEROES_BIT_LENGTH = DATA_BIT_LENGTH - 1; + const auto LEADING_ZEROES_BIT_LENGTH = DATA_BIT_LENGTH - 1; // worst case (for 32-bit value): // 11 + 5 bits of leading zeroes bit-size + 5 bits of data bit-size + non-zero data bits. diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 269e174f5c0a6..e2f6a43dbdf42 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -5,11 +5,37 @@ #include #include #include +#include + +#include namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; +} + +namespace +{ + +/// A codec never reserves less than its own input, so a result below the input means the UInt32 +/// arithmetic wrapped. The check does not depend on where inside the callee the wrap happened. +UInt32 getCheckedReserveSize(const CompressionCodecPtr & codec, UInt32 size, size_t codec_index, size_t codecs_count) +{ + UInt32 reserve_size = codec->getCompressedReserveSize(size); + if (reserve_size < size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: the size reserved for compressing {} bytes overflows 4 GiB " + "at codec {} of {} ({}). Use fewer codecs.", + size, codec_index + 1, codecs_count, codec->getCodecDesc()->formatForErrorMessage()); + return reserve_size; +} + +} + CompressionCodecMultiple::CompressionCodecMultiple(Codecs codecs_) : codecs(codecs_) { @@ -33,16 +59,36 @@ void CompressionCodecMultiple::updateHash(SipHash & hash) const UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const { + /// doCompressData stores the number of codecs in one byte, and doDecompressData reads it back + /// from there, so a longer chain would write a part that cannot be read. + if (codecs.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: {}. The number of codecs is stored in one byte, " + "so at most {} are supported.", + codecs.size(), static_cast(std::numeric_limits::max())); + UInt32 compressed_size = uncompressed_size; - for (const auto & codec : codecs) - compressed_size = codec->getCompressedReserveSize(compressed_size); + for (size_t idx = 0; idx < codecs.size(); ++idx) + compressed_size = getCheckedReserveSize(codecs[idx], compressed_size, idx, codecs.size()); /// TotalCodecs ByteForEachCodec data - return static_cast(sizeof(UInt8) + codecs.size() + compressed_size); + size_t total_size = sizeof(UInt8) + codecs.size() + compressed_size; + /// getCompressedReserveSize adds getHeaderSize() to this in UInt32, so leave room for it. + static constexpr size_t max_total_size = std::numeric_limits::max() - ICompressionCodec::getHeaderSize(); + if (total_size > max_total_size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: the size reserved for compressing {} bytes ({}) exceeds 4 GiB. " + "Use fewer codecs.", + uncompressed_size, total_size); + + return static_cast(total_size); } UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 source_size, char * dest) const { + /// The caller sized dest from getMaxCompressedDataSize(source_size). + const UInt32 dest_size = getMaxCompressedDataSize(source_size); + PODArray compressed_buf; PODArray uncompressed_buf(source, source + source_size); @@ -53,7 +99,7 @@ UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 sour { const auto codec = codecs[idx]; dest[codecs_byte_pos] = codec->getMethodByte(); - compressed_buf.resize(codec->getCompressedReserveSize(source_size)); + compressed_buf.resize(getCheckedReserveSize(codec, source_size, idx, codecs.size())); UInt32 size_compressed = codec->compress(uncompressed_buf.data(), source_size, compressed_buf.data()); @@ -61,9 +107,15 @@ UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 sour source_size = size_compressed; } + /// source_size is now each codec's actual output, computed independently of the bounds above. + size_t written_size = sizeof(UInt8) + codecs.size() + source_size; + if (written_size > dest_size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Compressed data of size {} does not fit the reserved buffer of size {}", written_size, dest_size); + memcpy(&dest[1 + codecs.size()], uncompressed_buf.data(), source_size); - return static_cast(1 + codecs.size() + source_size); + return static_cast(written_size); } UInt32 CompressionCodecMultiple::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 08c56cf9827df..87f9d8f8f00cb 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -21,8 +21,11 @@ #include #include #include +#include #include +#include + namespace DB { @@ -124,6 +127,22 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedASTImpl( { ASTPtr codecs_descriptions = make_intrusive(); + /// A codec that depends on the data type resolves differently per substream, and every + /// substream is compressed with its own chain, so there is one chain per substream. + size_t num_substreams = 0; + if (column_type) + { + ISerialization::StreamCallback count_callback = [&](const auto & substream_path) + { + if (ISerialization::isSpecialCompressionAllowed(substream_path)) + ++num_substreams; + }; + column_type->getDefaultSerialization()->enumerateStreams(count_callback, column_type); + } + /// A codec resolved without a data type is one chain, and so is a type whose substreams all + /// refuse a special codec. + VectorWithMemoryTracking codec_chains(num_substreams ? num_substreams : 1); + bool with_compression_codec = false; bool with_none_codec = false; std::optional first_generic_compression_codec_pos; @@ -167,6 +186,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedASTImpl( if (column_type) { CompressionCodecPtr prev_codec; + size_t substream_index = 0; ISerialization::StreamCallback callback = [&](const auto & substream_path) { chassert(!substream_path.empty()); @@ -175,6 +195,12 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedASTImpl( const auto & last_type = substream_path.back().data.type; result_codec = getImpl(codec_family_name, codec_arguments, last_type.get()); + /// Enumeration order is the same for every codec of the chain, so the + /// index identifies the substream. + if (substream_index < codec_chains.size()) + codec_chains[substream_index].push_back(result_codec); + ++substream_index; + /// Case for column Tuple, which compressed with codec which depends on data type, like Delta. /// We cannot substitute parameters for such codecs. if (prev_codec && prev_codec->getHash() != result_codec->getHash()) @@ -242,6 +268,11 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedASTImpl( codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); } + /// A codec that was not resolved per substream is the same one for all of them. + for (auto & chain : codec_chains) + if (chain.size() == i) + chain.push_back(result_codec); + with_compression_codec |= result_codec->isCompression(); with_none_codec |= result_codec->isNone(); @@ -262,6 +293,33 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedASTImpl( if (sanity_check) { + /// CompressionCodecMultiple stores the number of codecs in a single byte, so a longer chain + /// describes a part that cannot be read back. Being a sanity check, this is not enforced on + /// the metadata-load path, where an already stored table must stay loadable. + if (codecs_descriptions->children.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Too many codecs in the codec chain: {}. The number of codecs is stored in one byte, " + "so at most {} are supported.", + codecs_descriptions->children.size(), static_cast(std::numeric_limits::max())); + + /// A codec never reserves less than its input, so a reserve below the input means the + /// UInt32 compounding wrapped. One byte is the weakest block there is: a chain that + /// wraps on it cannot compress a block of any size. + for (const auto & chain : codec_chains) + { + UInt32 reserve_size = 1; + for (size_t i = 0; i < chain.size(); ++i) + { + const UInt32 next_reserve_size = chain[i]->getCompressedReserveSize(reserve_size); + if (next_reserve_size < reserve_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Too many codecs in the codec chain: {}. The size they reserve for compressing a block " + "overflows 4 GiB at codec {} ({}), so no block could be compressed. Use fewer codecs.", + chain.size(), i + 1, chain[i]->getCodecDesc()->formatForErrorMessage()); + reserve_size = next_reserve_size; + } + } + if (codecs_descriptions->children.size() > 1 && with_none_codec) throw Exception(ErrorCodes::BAD_ARGUMENTS, "It does not make sense to have codec NONE along with other compression codecs: {}. " diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 693e332bb674d..9a8c5912c7baf 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -2139,9 +2139,11 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const && (command.default_kind == ColumnDefaultKind::Default || command.default_kind == ColumnDefaultKind::Materialized); if (all_columns.hasAlias(column_name) && !becomes_physical) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); + /// The type is optional here, and a codec can resolve differently per type, so + /// validate against the type the column will have, as `apply` does. CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( command.codec, - command.data_type, + command.data_type ? command.data_type : all_columns.get(column_name).type, codec_validation_settings); } auto column_default = all_columns.getDefault(column_name); diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference new file mode 100644 index 0000000000000..15a39899594a5 --- /dev/null +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference @@ -0,0 +1,14 @@ +5000 0 +0 +4 +5000 0 +0 +2500 0 +0 +5000 0 +0 +5000 0 +0 +0 +0 +5000 0 diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql new file mode 100644 index 0000000000000..f047a1d4dc1af --- /dev/null +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql @@ -0,0 +1,176 @@ +-- Tags: no-fasttest, no-parallel +-- Tag no-fasttest: a codec chain long enough to overflow a 32-bit reserve needs a second or two. +-- Tag no-parallel: the 104-stage arm below allocates just under 4 GiB against a per-user memory +-- limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. +-- Random settings limits: min_columns_to_activate_adaptive_write_buffer=(3, None) + +-- CompressionCodecMultiple compounded each stage's reserved size in a UInt32 without checking for +-- overflow. Every codec reserves more than its input, so a long enough chain wrapped the reserve +-- below its own input, and the destination buffer was then sized from the wrapped number while the +-- real payload kept growing: an out-of-bounds heap write, and a server crash. Such a chain is +-- rejected when the table is created now. A chain of more than 255 codecs is rejected there too: +-- the codec count is stored in a single byte, so a longer chain used to write an unreadable part. + +-- The 104-stage chain below reserves just under 4 GiB for one block, which is what this test is +-- about, and that is most of the per-query limit CI's default profile sets. This test measures a +-- size bound, not memory consumption, so lift the limit rather than tune it. +SET max_memory_usage = 0; + +DROP TABLE IF EXISTS t_codec_chain_overflow; +DROP TABLE IF EXISTS t_codec_chain_fpc_max; +DROP TABLE IF EXISTS t_codec_chain_fpc_over; +DROP TABLE IF EXISTS t_codec_chain_gorilla_wide; +DROP TABLE IF EXISTS t_codec_chain_gorilla_max; +DROP TABLE IF EXISTS t_codec_chain_gorilla_over; +DROP TABLE IF EXISTS t_codec_chain_gorilla64_max; +DROP TABLE IF EXISTS t_codec_chain_gorilla64_over; +DROP TABLE IF EXISTS t_codec_chain_dd_max; +DROP TABLE IF EXISTS t_codec_chain_dd_over; +DROP TABLE IF EXISTS t_codec_chain_short; +DROP TABLE IF EXISTS t_codec_chain_255; +DROP TABLE IF EXISTS t_codec_chain_256; +DROP TABLE IF EXISTS t_codec_chain_256_meta; +DROP TABLE IF EXISTS t_codec_chain_meta_alter; +DROP TABLE IF EXISTS t_codec_chain_tuple_mixed; +DROP TABLE IF EXISTS t_codec_chain_tuple_wide; + +-- 150 FPC stages. Compressing a block with this chain is impossible whatever the block size, so +-- the chain is rejected when the table is created. Before the fix the table was created and the +-- first write crashed the server with an out-of-bounds heap write. +CREATE TABLE t_codec_chain_overflow (x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- 104 FPC stages: the longest chain whose reserve still fits. It must keep working and round-trip. +-- The stage the reserve wraps at is a function of the compressed block size, so the three tables +-- below pin everything that sets it: 5000 Float32 then form one 20000-byte block, wrapping at +-- stage 105. A smaller block wraps later, which would make the next arm accept its chain. +CREATE TABLE t_codec_chain_fpc_max (i UInt32, x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_fpc_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_fpc_max; + +-- One stage more, on the same fixture, is the first chain that is rejected. +CREATE TABLE t_codec_chain_fpc_over (i UInt32, x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_fpc_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_fpc_over; + +-- Gorilla's reserve per item is a function of the column's width, so a chain of it has a boundary +-- per width. Both boundaries are pinned below, because whichever width a process compresses first +-- is the one that used to fix the reserve for every other width, and a test cannot choose that. +CREATE TABLE t_codec_chain_gorilla_wide (d Float64 CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_codec_chain_gorilla_wide SELECT number / 7 FROM numbers(4); +SELECT count() FROM t_codec_chain_gorilla_wide; + +-- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 29 stages is the +-- longest that still fits; it must keep working too. +CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_gorilla_max; + +-- The reserve is compounded the same way whatever the codec, so the boundary below is Gorilla's own +-- and one stage past it is rejected, as for FPC above. The three arms that follow pin the sibling +-- formulas, so that rejection cannot silently narrow to FPC alone. +CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_gorilla_over; + +-- The same boundary for a Float64 Gorilla column: 48 stages fit, 49 do not. A chain of one width +-- must be judged on its own width, so these two keep their verdicts however the Float32 arms above +-- left the process. +CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla64_max SELECT number, number / 7 FROM numbers(2500); +SELECT count(), countIf(d != i / 7) FROM t_codec_chain_gorilla64_max; + +CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla64_over SELECT number, number / 7 FROM numbers(2500); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_gorilla64_over; + +-- DoubleDelta reserves more per stage still: 13 is the longest chain that fits, 14 is rejected. +CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_dd_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_dd_max; + +CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_dd_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_dd_over; + +-- 255 codecs is the longest chain the one-byte count can describe. Delta does not expand, so the +-- reserve stays small and only the count matters here. +CREATE TABLE t_codec_chain_255 (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i; +INSERT INTO t_codec_chain_255 SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_255; + +-- One codec more used to write a part that could not be read back. +CREATE TABLE t_codec_chain_256 (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i SETTINGS allow_suspicious_codecs = 1; +INSERT INTO t_codec_chain_256 SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_256; + +-- A chain the one-byte count cannot describe is rejected when the metadata is created, so that it +-- cannot be persisted and replicated as a codec no replica can ever write with. This check is a +-- sanity check, so the arms above, which set allow_suspicious_codecs, still reach the write path. +CREATE TABLE t_codec_chain_256_meta (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_codec_chain_meta_alter (i UInt32, x Float32, w Float64) ENGINE = MergeTree ORDER BY i; +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter ADD COLUMN y Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } + +-- The longest chain the count can describe is still accepted there. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; +SELECT count() FROM t_codec_chain_meta_alter; + +-- The reserve rule applies to ALTER as well, for both forms that carry a codec. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter ADD COLUMN z Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } + +-- A codec whose reserve depends on the column width resolves per substream, so a heterogeneous +-- column has one chain per substream and each of them must fit. Either element order is rejected +-- because the Float32 substream alone cannot compress a block. +CREATE TABLE t_codec_chain_tuple_mixed (x Tuple(Float32, Float64) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_codec_chain_tuple_mixed (x Tuple(Float64, Float32) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- The same chain on a column all of whose substreams are 8 bytes wide reserves far less and stays +-- accepted, so the check above is not a blanket rejection of the chain length. +CREATE TABLE t_codec_chain_tuple_wide (x Tuple(Float64, Float64) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); + +-- MODIFY COLUMN may omit the type, and the codec then resolves against the type the column +-- already has, so the same chain is judged by that width and not by the null-type default. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN w CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); +SELECT count() FROM t_codec_chain_meta_alter; + +-- An ordinary short chain is unaffected. +CREATE TABLE t_codec_chain_short (i UInt32, x Float32 CODEC(Delta, ZSTD)) ENGINE = MergeTree ORDER BY i; +INSERT INTO t_codec_chain_short SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_short; + +DROP TABLE t_codec_chain_fpc_max; +DROP TABLE t_codec_chain_fpc_over; +DROP TABLE t_codec_chain_gorilla_wide; +DROP TABLE t_codec_chain_gorilla_max; +DROP TABLE t_codec_chain_gorilla_over; +DROP TABLE t_codec_chain_gorilla64_max; +DROP TABLE t_codec_chain_gorilla64_over; +DROP TABLE t_codec_chain_dd_max; +DROP TABLE t_codec_chain_dd_over; +DROP TABLE t_codec_chain_short; +DROP TABLE t_codec_chain_255; +DROP TABLE t_codec_chain_256; +DROP TABLE t_codec_chain_meta_alter; +DROP TABLE t_codec_chain_tuple_wide; From 227a9027799bb1475216d43c227a9f27b6eec32e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 12:06:05 +0000 Subject: [PATCH 41/81] Backport #124075 to 26.8: Leave only loadable objects behind in `05316_named_collection_dictionary_override_access` --- ...d_collection_dictionary_override_access.sh | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh index 2039d59fd0bdb..9bbc91a03f3eb 100755 --- a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh +++ b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh @@ -10,6 +10,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_dict" +nc_locked="${CLICKHOUSE_TEST_UNIQUE_NAME}_locked" nc_load="${CLICKHOUSE_TEST_UNIQUE_NAME}_load" nc_mysql="${CLICKHOUSE_TEST_UNIQUE_NAME}_mysql" nc_url="${CLICKHOUSE_TEST_UNIQUE_NAME}_url" @@ -17,7 +18,19 @@ user="${CLICKHOUSE_TEST_UNIQUE_NAME}_user" function cleanup() { + # A client killed inside attach_is_rejected leaves its table detached permanently, which DROP TABLE does not see. + for table in $(${CLICKHOUSE_CLIENT} --query "SELECT table FROM system.detached_tables + WHERE database = currentDatabase() AND table IN ('table_settings_override', 'table_alias_override')"); do + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + ALTER NAMED COLLECTION IF EXISTS $nc_mysql SET connection_pool_size = 2 OVERRIDABLE; + ALTER NAMED COLLECTION IF EXISTS $nc_url SET http_method = 'POST' OVERRIDABLE; + ATTACH TABLE $table; + " + done + # Neither ignore nor fuzz these drops: each object must be gone before the named collection it references. ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; DROP DICTIONARY IF EXISTS dict_override; DROP DICTIONARY IF EXISTS dict_add_key; DROP DICTIONARY IF EXISTS dict_alias; @@ -29,6 +42,7 @@ function cleanup() DROP TABLE IF EXISTS dict_source_b; DROP USER IF EXISTS $user; DROP NAMED COLLECTION IF EXISTS $nc; + DROP NAMED COLLECTION IF EXISTS $nc_locked; DROP NAMED COLLECTION IF EXISTS $nc_load; DROP NAMED COLLECTION IF EXISTS $nc_mysql; DROP NAMED COLLECTION IF EXISTS $nc_url; @@ -46,6 +60,9 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " CREATE NAMED COLLECTION $nc AS host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; + CREATE NAMED COLLECTION $nc_locked AS + host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', + db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a' NOT OVERRIDABLE; CREATE NAMED COLLECTION $nc_load AS host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; @@ -54,6 +71,7 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " GRANT SOURCES ON *.* TO $user; GRANT CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${CLICKHOUSE_DATABASE}.* TO $user; GRANT NAMED COLLECTION ON $nc TO $user; + GRANT NAMED COLLECTION ON $nc_locked TO $user; " echo 'Overriding a stored key in a dictionary source requires the secrets privilege' @@ -83,10 +101,11 @@ ${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " " echo 'A NOT OVERRIDABLE key cannot be overridden even with the secrets privilege' -${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc SET table = 'dict_source_a' NOT OVERRIDABLE" +# A separate collection: no stored dictionary overrides a key while it is locked, so a restart can load what the test leaves. +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $nc_locked TO $user" ${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " CREATE DICTIONARY dict_locked (id UInt64, value UInt64) PRIMARY KEY id - SOURCE(CLICKHOUSE(NAME $nc TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError BAD_ARGUMENTS } + SOURCE(CLICKHOUSE(NAME $nc_locked TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError BAD_ARGUMENTS } " echo 'A dictionary created with an override by a privileged user loads in the background' @@ -102,7 +121,9 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " " echo 'A stored table whose override became NOT OVERRIDABLE is rejected when it is attached' +# The fuzzer stays off for all DDL of these tables: a __fuzz_N clone would stay attached while its key is locked. ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; CREATE NAMED COLLECTION $nc_mysql AS host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', connection_pool_size = 2; @@ -113,28 +134,30 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " " # A table that overrides a key which is locked afterwards cannot be attached: the lock is checked on every load. +# The table stays detached permanently while its key is locked, so a server restart never loads it. function attach_is_rejected() { local table=$1 - local key=$2 - ${CLICKHOUSE_CLIENT} --query "DETACH TABLE $table" - if error=$(${CLICKHOUSE_CLIENT} --query "ATTACH TABLE $table" 2>&1); then + local collection=$2 + local key=$3 + local value=$4 + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + DETACH TABLE $table PERMANENTLY; + ALTER NAMED COLLECTION $collection SET $key = $value NOT OVERRIDABLE; + " + if error=$(${CLICKHOUSE_CLIENT} --multiquery --query "SET ast_fuzzer_any_query = 0; ATTACH TABLE $table" 2>&1); then echo "Expected the attach of $table to be rejected" exit 1 fi echo "$error" | grep -o "Override not allowed for '$key'" | head -1 echo "$error" | grep -o 'BAD_ARGUMENTS' | head -1 + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + ALTER NAMED COLLECTION $collection SET $key = $value OVERRIDABLE; + ATTACH TABLE $table; + " } -${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 NOT OVERRIDABLE" -attach_is_rejected table_settings_override connection_pool_size -${CLICKHOUSE_CLIENT} --multiquery --query " - ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 OVERRIDABLE; - ATTACH TABLE table_settings_override; - ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' NOT OVERRIDABLE; -" -attach_is_rejected table_alias_override http_method -${CLICKHOUSE_CLIENT} --multiquery --query " - ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' OVERRIDABLE; - ATTACH TABLE table_alias_override; -" +attach_is_rejected table_settings_override "$nc_mysql" connection_pool_size 2 +attach_is_rejected table_alias_override "$nc_url" http_method "'POST'" From fdaf943d5ce653fff45c83f942a3d91077670344 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 13:33:14 +0000 Subject: [PATCH 42/81] Backport #120407 to 26.8: [SECURITY] Mask credentials embedded in HTTP URIs --- src/Common/maskURIPassword.h | 14 ++ src/IO/HTTPCommon.cpp | 50 ++++++- src/IO/HTTPCommon.h | 19 ++- src/IO/ReadWriteBufferFromHTTP.cpp | 26 ++-- src/IO/WriteBufferFromHTTP.cpp | 6 +- src/Storages/StorageURL.cpp | 10 +- .../StorageSystemSchemaInferenceCache.cpp | 8 +- ...5212_http_uri_credentials_masked.reference | 12 ++ .../05212_http_uri_credentials_masked.sh | 135 ++++++++++++++++++ 9 files changed, 260 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/05212_http_uri_credentials_masked.reference create mode 100755 tests/queries/0_stateless/05212_http_uri_credentials_masked.sh diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index 29038c229fb73..b49fd22240075 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -213,4 +213,18 @@ inline bool maskPresignedURLParameters(std::string & url) return true; } +/** Mask both credential carriers of a URL: the userinfo (`user:password@`) and the values of the + * presigned-URL query parameters. Returns whether anything was masked. + * + * This is the masking to use for a whole HTTP/S3 URL. Scanning for a presigned parameter is only + * sound on a value that is one whole URL (see `maskPresignedURLParameters`), so the argument must be + * one URL, not a URL embedded in a longer string. + */ +inline bool maskURICredentials(std::string & url) +{ + bool masked = maskURIUserinfo(url); + masked |= maskPresignedURLParameters(url); + return masked; +} + } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 14d7fd8d4db06..824bc30d4c031 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -3,6 +3,9 @@ #include #include #include +#include + +#include #include "config.h" @@ -81,11 +84,36 @@ std::istream * receiveResponse( Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, const bool allow_redirects) { auto & istr = session.receiveResponse(response); - assertResponseIsOk(request.getURI(), response, istr, allow_redirects); + assertResponseIsOk(request.getURI(), response, istr, allow_redirects, requestCredentialSecrets(request)); return &istr; } -void assertResponseIsOk(const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, const bool allow_redirects) +Strings requestCredentialSecrets(const Poco::Net::HTTPRequest & request) +{ + if (!request.has("Authorization")) + return {}; + + /// `getCredentials` splits the scheme off the header value the way an HTTP peer does: the scheme + /// is matched case-insensitively, and a field value may begin with whitespace (RFC 9110, 5.5), + /// which `auth_header = 'Authorization: Bearer '` leaves in front of the scheme. + std::string scheme; + std::string auth_info; + request.getCredentials(scheme, auth_info); + Poco::trimInPlace(auth_info); + /// Only a bearer token is recoverable from the header, and it is a secret as a whole. A `Basic` + /// header holds base64 of `user:password` and is deliberately not decoded here: its value is + /// user input whenever it came from `headers()`, decoding throws on anything outside the base64 + /// alphabet, and the user name of a pair is an identifier rather than a credential. A password + /// is therefore not scrubbed from a response body; no peer is known to echo one back. + if (Poco::icompare(scheme, "Bearer") == 0 && !auth_info.empty()) + return {auth_info}; + + return {}; +} + +void assertResponseIsOk( + const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, const bool allow_redirects, + const Strings & body_secrets) { auto status = response.getStatus(); @@ -102,7 +130,7 @@ void assertResponseIsOk(const String & uri, Poco::Net::HTTPResponse & response, std::string body; Poco::StreamCopier::copyToString(istr, body); - throw HTTPException(code, uri, status, response.getReason(), body); + throw HTTPException(code, uri, status, response.getReason(), body, body_secrets); } } @@ -111,13 +139,25 @@ Exception HTTPException::makeExceptionMessage( const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status, const std::string & reason, - const std::string & body) + const std::string & body, + const Strings & body_secrets) { + std::string masked_uri = uri; + maskURICredentials(masked_uri); + + /// The response body is remote content that can reflect the request's own credentials back, which + /// a server may echo verbatim (e.g. an auth error naming the user). Scrub those exact credential + /// strings, keeping the rest of the body - a useful diagnostic that does not carry a credential. + std::string masked_body = body; + for (const auto & secret : body_secrets) + if (!secret.empty()) + boost::replace_all(masked_body, secret, "[HIDDEN]"); + return Exception(code, "Received error from remote server {}. " "HTTP status code: {} '{}', " "body length: {} bytes, body: '{}'", - uri, static_cast(http_status), reason, body.length(), body); + masked_uri, static_cast(http_status), reason, body.length(), masked_body); } } diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index 18b808b5cb922..fd731eaf5f7ce 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -28,9 +29,10 @@ class HTTPException : public Exception const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status_, const std::string & reason, - const std::string & body + const std::string & body, + const Strings & body_secrets = {} ) - : Exception(makeExceptionMessage(code, uri, http_status_, reason, body)) + : Exception(makeExceptionMessage(code, uri, http_status_, reason, body, body_secrets)) , http_status(http_status_) {} @@ -47,7 +49,8 @@ class HTTPException : public Exception const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status, const std::string & reason, - const std::string & body); + const std::string & body, + const Strings & body_secrets); const char * name() const noexcept override { return "DB::HTTPException"; } const char * className() const noexcept override { return "DB::HTTPException"; } @@ -81,7 +84,15 @@ bool isRetriableHTTPError(Poco::Net::HTTPResponse::HTTPStatus http_status) noexc std::istream * receiveResponse( Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, bool allow_redirects); +/// Returns the Bearer token carried by a request's `Authorization` header, so a caller can scrub it +/// from an error response body that reflects it back. A `Basic` header is not decoded, so a password +/// is not scrubbed. Empty when there is no bearer token. +Strings requestCredentialSecrets(const Poco::Net::HTTPRequest & request); + +/// `body_secrets` are credential strings to scrub from the reflected response body (see +/// `requestCredentialSecrets`): a remote error body can echo the request's own credentials. void assertResponseIsOk( - const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false, + const Strings & body_secrets = {}); } diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index da0874411694b..33ae4e0e9b6d0 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,15 @@ Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPRespons return location_uri; } +/// A URI that ends up in a log line or an exception message must never carry credentials. Convenience +/// overload for a `Poco::URI`: masks the userinfo and presigned-URL parameters via the shared helper. +std::string maskedURI(const Poco::URI & uri) +{ + std::string masked = uri.toString(); + DB::maskURICredentials(masked); + return masked; +} + class ReadBufferFromSessionResponse : public DB::ReadBufferFromIStream { private: @@ -163,7 +173,7 @@ bool ReadWriteBufferFromHTTP::checkIfActuallySeekable() String ReadWriteBufferFromHTTP::getFileName() const { - return initial_uri.toString(); + return maskedURI(initial_uri); } void ReadWriteBufferFromHTTP::getHeadResponse(Poco::Net::HTTPResponse & response) @@ -276,7 +286,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( auto & resp_stream = session->receiveResponse(response); - assertResponseIsOk(current_uri.toString(), response, resp_stream, allow_redirects); + assertResponseIsOk(maskedURI(current_uri), response, resp_stream, allow_redirects, requestCredentialSecrets(request)); return ReadWriteBufferFromHTTP::CallResult(std::move(session), resp_stream); } @@ -297,7 +307,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( " You can {} redirects by changing the setting 'max_http_get_redirects'." " Example: `SET max_http_get_redirects = 10`." " Redirects are restricted to prevent possible attack when a malicious server redirects to an internal resource, bypassing the authentication or firewall.", - initial_uri.toString(), max_redirects ? "increase the allowed maximum number of" : "allow"); + maskedURI(initial_uri), max_redirects ? "increase the allowed maximum number of" : "allow"); if (redirect_callback) redirect_callback(current_uri, uri_redirect); @@ -373,7 +383,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, "Failed to make request to '{}'{}. " "Error: '{}'. " "Failed at try {}/{}.", - initial_uri.toString(), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + maskedURI(initial_uri), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", maskedURI(current_uri)), error_message, attempt, read_settings.http_settings.max_tries); @@ -390,7 +400,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, "Error: {}. " "Failed at try {}/{}. " "Will retry with current backoff wait is {}/{} ms.", - initial_uri.toString(), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + maskedURI(initial_uri), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", maskedURI(current_uri)), error_message, attempt + 1, read_settings.http_settings.max_tries, milliseconds_to_wait, read_settings.http_settings.retry_max_backoff_ms); @@ -428,7 +438,7 @@ std::unique_ptr ReadWriteBufferFromHTTP::initialize() /// it is retriable error throw HTTPException( ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, - current_uri.toString(), + maskedURI(current_uri), Poco::Net::HTTPResponse::HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, response.getReason(), explanation); @@ -548,13 +558,13 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co String explanation = fmt::format( "When reading with readBigAt {}." "Cannot read with range: [{}, {}] (response status: {}, reason: {}), will retry", - initial_uri.toString(), + maskedURI(initial_uri), *range.begin, *range.end, toString(response.getStatus()), response.getReason()); throw HTTPException( ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, - current_uri.toString(), + maskedURI(current_uri), Poco::Net::HTTPResponse::HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, response.getReason(), explanation); diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index dc95d582fd8a0..4edb13d7bafea 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -46,7 +47,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.set("Content-Type", content_type); } - LOG_TRACE((getLogger("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); + /// Mask credentials before logging: a URL sink would otherwise log them on every INSERT. + std::string uri_for_logging = uri.toString(); + maskURICredentials(uri_for_logging); + LOG_TRACE((getLogger("WriteBufferToHTTP")), "Sending request to {}", uri_for_logging); ProfileEvents::increment(ProfileEvents::WriteBufferFromHTTPRequestsSent); ostr = &session->sendRequest(request); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index fca4d854d2800..9f62881aabc25 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include @@ -994,7 +995,14 @@ namespace format = format_name; } - String getLastFilePath() const override { return current_url_option; } + String getLastFilePath() const override + { + /// `current_url_option` is kept raw for requests and schema cache keys, but the diagnostic + /// path can reach the client, server log and `system.query_log`, so mask credentials here. + String masked = current_url_option; + maskURICredentials(masked); + return masked; + } bool supportsLastReadBufferRecreation() const override { return true; } diff --git a/src/Storages/System/StorageSystemSchemaInferenceCache.cpp b/src/Storages/System/StorageSystemSchemaInferenceCache.cpp index cf5efe8a84783..1cbdc46dcb788 100644 --- a/src/Storages/System/StorageSystemSchemaInferenceCache.cpp +++ b/src/Storages/System/StorageSystemSchemaInferenceCache.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -59,7 +60,12 @@ static void fillDataImpl(MutableColumns & res_columns, SchemaCache & schema_cach for (const auto & [key, schema_info] : s3_schema_cache_data) { res_columns[0]->insert(storage_name); - res_columns[1]->insert(key.source); + String source = key.source; + /// Only a URL can carry a credential. A `File` key is a local path, where the presigned + /// parameter scan would rewrite a '?' in a file name and swallow everything after it. + if (findURIAuthority(source) != String::npos) + maskURICredentials(source); + res_columns[1]->insert(source); res_columns[2]->insert(key.format); res_columns[3]->insert(key.additional_format_info); res_columns[4]->insert(schema_info.registration_time); diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference new file mode 100644 index 0000000000000..229ae0059879a --- /dev/null +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference @@ -0,0 +1,12 @@ +url_function: OK masked +url_status_failure: OK masked +insert_presigned: OK masked +dictionary_reload: OK masked +show_create: OK masked +schema_inference_cache: OK masked +schema_inference_error: OK masked +basic_header_passthrough: OK +body_not_overmasked: OK +bearer_body_scrubbed_plain: OK +bearer_body_scrubbed_leading_space: OK +bearer_body_scrubbed_lower_case: OK diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh new file mode 100755 index 0000000000000..92cfea296769d --- /dev/null +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A credential embedded in an HTTP URI must never appear in an error message, in SHOW CREATE output, +# or in system.query_log. Only the masked form scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for +# a presigned URL) may be shown. + +PW="pwleakprobe9f2a" +URI="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping" +USERINFO_SHAPE="[HIDDEN]@${CLICKHOUSE_HOST}" + +# Reads text from stdin and asserts the credential was present but masked: the cleartext secret is +# absent and the required masked shape is present. Asserting the shape (not just that "[HIDDEN]" appears +# somewhere) means a text that never carried the credential fails, so the check cannot pass vacuously. +assert_shape() { + local label="$1" secret="$2" shape="$3" text + text=$(cat) + if echo "$text" | grep -qF "$secret"; then + echo "$label: FAIL cleartext" + elif echo "$text" | grep -qF "$shape"; then + echo "$label: OK masked" + else + echo "$label: FAIL uri absent" + fi +} + +# 1. url() table function: the /ping response ("Ok.") fails to parse as CSV, so the URI is appended to +# the exception as "(in file/uri ...)". Grep that line out of the exception - the client also echoes +# the user's own submitted query, which legitimately contains what the user typed. +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('${URI}', 'CSV', 'id UInt64, val String')" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "url_function" "$PW" "$USERINFO_SHAPE" + +# 1b. An HTTP status failure (non-2xx) is reported by assertResponseIsOk as "Received error from +# remote server ", a different code path than the CSV-parse suffix above. A request to an +# unknown path returns 404, so the URI in that exception must also be masked. +URI_404="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/no_such_handler_${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('${URI_404}', 'CSV', 'id UInt64, val String')" 2>&1 \ + | grep -F 'Received error from remote server' | assert_shape "url_status_failure" "$PW" "$USERINFO_SHAPE" + +# 1c. INSERT INTO url() writes through WriteBufferFromHTTP - the only path here that does - and a +# presigned URL carries its credential in the query parameters, not the userinfo. A request with a +# missing ?database returns 404 whose body does not echo the URI, so the signature can only appear +# through the URI in the exception, which must be masked to X-Amz-Signature=[HIDDEN]. +SIG="sigprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +URI_SIG="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?database=no_such_db_${CLICKHOUSE_TEST_UNIQUE_NAME}&X-Amz-Signature=${SIG}" +${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION url('${URI_SIG}', 'CSV', 'id UInt64') VALUES (1)" 2>&1 \ + | grep -F 'Received error from remote server' | assert_shape "insert_presigned" "$SIG" "X-Amz-Signature=[HIDDEN]" + +# 2. A dictionary whose HTTP source URL carries credentials. +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS dict_uri_leak" +${CLICKHOUSE_CLIENT} --query "CREATE DICTIONARY dict_uri_leak (id UInt64, val String) PRIMARY KEY id SOURCE(HTTP(url '${URI}' format 'CSV')) LAYOUT(FLAT(SIZE_IN_CELLS 100)) LIFETIME(0)" + +# 2a. Reloading fails to parse; the URI must be masked in the exception. +${CLICKHOUSE_CLIENT} --query "SYSTEM RELOAD DICTIONARY dict_uri_leak" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "dictionary_reload" "$PW" "$USERINFO_SHAPE" + +# 2b. SHOW CREATE must mask the password. +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DICTIONARY dict_uri_leak" 2>&1 | assert_shape "show_create" "$PW" "$USERINFO_SHAPE" + +# 3. system.query_log must store neither the query text nor the exception with the cleartext password. +# Print the offending rows rather than a count: on success this selects nothing (the reference has +# no line here), and any row printed on failure is the culprit - its query and exception show what +# leaked. The needle is split so that this checking query does not itself contain the contiguous +# secret. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT type, query, exception + FROM system.query_log + WHERE event_date >= yesterday() + AND current_database = currentDatabase() + AND (query LIKE '%' || 'pwleakprobe' || '9f2a%' OR exception LIKE '%' || 'pwleakprobe' || '9f2a%') + FORMAT Vertical" + +# 4. Schema inference stores the source URL in system.schema_inference_cache; the credential in its +# userinfo must be masked there too. Inferring a schema from a credentialed /ping URL (which needs +# no auth and whose "Ok." body is inferrable as one column) populates the cache. +SIC_SECRET="sicpwprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://leakuser:${SIC_SECRET}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'CSV')" >/dev/null 2>&1 +${CLICKHOUSE_CLIENT} --query "SELECT source FROM system.schema_inference_cache WHERE storage = 'URL'" \ + | assert_shape "schema_inference_cache" "$SIC_SECRET" "[HIDDEN]@${CLICKHOUSE_HOST}" + +# 4b. When schema inference fails, the source URL is appended to the error as "(in file/uri ...)", and it +# must be masked there too. The "Ok." body of /ping is not JSON, so inferring JSONEachRow fails. +SIE_SECRET="siepwprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "DESC url('http://leakuser:${SIE_SECRET}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'JSONEachRow')" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "schema_inference_error" "$SIE_SECRET" "(in file/uri http://[HIDDEN]@${CLICKHOUSE_HOST}" + +# 5. A header value is never decoded as a credential. An 'Authorization: Basic ' whose payload +# is not valid base64 (here supplied through headers()) must reach the endpoint verbatim, not fail +# while the request is inspected for secrets to scrub. /ping needs no auth and answers "Ok.". +A_OUT=$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'LineAsString', 'line String', headers('Authorization' = 'Basic ghp_AbCd1234'))" 2>&1) +if echo "$A_OUT" | grep -qF 'Ok.'; then echo "basic_header_passthrough: OK"; else echo "basic_header_passthrough: FAIL"; fi + +# 6. A URL userinfo is sent as 'Authorization: Basic base64(user:password)'. Recovering the user name +# from that header to scrub the error body would replace a one-letter user ('a') everywhere it +# occurs, turning the remote "Authentication failed" body into "Authentic[HIDDEN]tion f[HIDDEN]iled". +# The body must be preserved, and the password must not appear. Authenticating against ClickHouse's +# own HTTP port with a bad password returns that body. Grep the "Received error from remote server" +# line out of the exception: the client also echoes the user's own submitted query, which +# legitimately contains the password the user typed. +B_OUT=$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://a:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'CSV', 'x UInt8')" 2>&1 \ + | grep -F 'Received error from remote server') +if echo "$B_OUT" | grep -qF "$PW"; then + echo "body_not_overmasked: FAIL cleartext" +elif echo "$B_OUT" | grep -qF 'Authentication failed'; then + echo "body_not_overmasked: OK" +else + echo "body_not_overmasked: FAIL mangled" +fi + +# 7. A bearer token that the remote server reflects in its error body must be scrubbed. The URL +# authenticates through its parameters, which take precedence over the 'Authorization' header, so the +# remote runs the query and throwIf echoes the token; concat() keeps the token out of the URI, so the +# body is its only carrier. Also covers the scheme spellings an HTTP peer accepts: a leading space and +# lower case. Grep the "Received error from remote server" line: the client echoes the submitted query. +TOK="tok${CLICKHOUSE_TEST_UNIQUE_NAME}" +for CASE in "plain|Bearer" "leading_space| Bearer" "lower_case|bearer"; do + LABEL="${CASE%%|*}" + SCHEME="${CASE#*|}" + T_OUT=$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?user=default&query=SELECT+throwIf(1,concat(''tok'',''${CLICKHOUSE_TEST_UNIQUE_NAME}''))', 'CSV', 'x UInt8', headers('Authorization' = '${SCHEME} ${TOK}')) SETTINGS http_max_tries = 1" 2>&1 \ + | grep -F 'Received error from remote server') + if echo "$T_OUT" | grep -qF "$TOK"; then + echo "bearer_body_scrubbed_${LABEL}: FAIL cleartext" + elif echo "$T_OUT" | grep -qF '[HIDDEN]'; then + echo "bearer_body_scrubbed_${LABEL}: OK" + else + echo "bearer_body_scrubbed_${LABEL}: FAIL no reflected body" + fi +done + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS dict_uri_leak" From 43b38c83b3ee7463904197d74f9f90099e47051b Mon Sep 17 00:00:00 2001 From: yariks5s Date: Mon, 19 Oct 2026 13:48:21 +0000 Subject: [PATCH 43/81] Fix the scope of the inherited definition check and drop the extra privilege check --- src/Interpreters/InterpreterCreateQuery.cpp | 57 +++++++++---------- ...cit_engine_requires_show_columns.reference | 2 - ...r_explicit_engine_requires_show_columns.sh | 28 --------- 3 files changed, 27 insertions(+), 60 deletions(-) delete mode 100644 tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference delete mode 100755 tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 97931dfc0803f..47fad29cdac17 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1536,36 +1536,36 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug."); } - } - if (storage_def) - { - /// check what the server stores: settings in this query replace the source settings, and - /// Null can replace an external engine - auto inherited = boost::static_pointer_cast(storage_def->clone()); - if (create.storage) + if (storage_def) { - /// inheritStorageFromSource keeps the clauses this query gives, so drop them here too - if (create.storage->partition_by) - inherited->reset(inherited->partition_by); - if (create.storage->primary_key) - inherited->reset(inherited->primary_key); - if (create.storage->order_by) - inherited->reset(inherited->order_by); - if (create.storage->sample_by) - inherited->reset(inherited->sample_by); - if (create.storage->ttl_table) - inherited->reset(inherited->ttl_table); - if (create.storage->unique_key) - inherited->reset(inherited->unique_key); - if (inherited->settings && create.storage->settings) - for (const auto & change : create.storage->settings->changes) - inherited->settings->changes.removeSetting(change.name); - } - replaceExternalEngineWithNullIfNeeded( - *inherited, getContext()->getSettingsRef()[Setting::restore_replace_external_engines_to_null]); + /// check what the server stores: settings in this query replace the source settings, and + /// Null can replace an external engine + auto inherited = boost::static_pointer_cast(storage_def->clone()); + if (create.storage) + { + /// inheritStorageFromSource keeps the clauses this query gives, so drop them here too + if (create.storage->partition_by) + inherited->reset(inherited->partition_by); + if (create.storage->primary_key) + inherited->reset(inherited->primary_key); + if (create.storage->order_by) + inherited->reset(inherited->order_by); + if (create.storage->sample_by) + inherited->reset(inherited->sample_by); + if (create.storage->ttl_table) + inherited->reset(inherited->ttl_table); + if (create.storage->unique_key) + inherited->reset(inherited->unique_key); + if (inherited->settings && create.storage->settings) + for (const auto & change : create.storage->settings->changes) + inherited->settings->changes.removeSetting(change.name); + } + replaceExternalEngineWithNullIfNeeded( + *inherited, getContext()->getSettingsRef()[Setting::restore_replace_external_engines_to_null]); - check_access_to_inherited_definition(*inherited); + check_access_to_inherited_definition(*inherited); + } } if (create.storage) @@ -3526,9 +3526,6 @@ BlockIO InterpreterCreateQuery::execute() /// the UUIDs above do create.as_database = getContext()->resolveDatabase(create.as_database); - if (create.storage && create.storage->engine) - getContext()->checkAccess(AccessType::SHOW_COLUMNS, create.as_database, create.as_table); - /// OLDEST_VERSION sends no settings, so a worker replaces nothing with Null. check /// what the worker builds, not what our settings give auto preflight_context = Context::createCopy(getContext()); diff --git a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference deleted file mode 100644 index 88c388e3e7f0d..0000000000000 --- a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.reference +++ /dev/null @@ -1,2 +0,0 @@ -without SHOW COLUMNS: -necessary to have the grant SHOW COLUMNS ON db.plain_src diff --git a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh b/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh deleted file mode 100755 index dc0aae0446d62..0000000000000 --- a/tests/queries/0_stateless/05399_create_as_on_cluster_explicit_engine_requires_show_columns.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Tags: no-replicated-database -# Tag no-replicated-database: a Replicated database does not allow ON CLUSTER - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -user="user_${CLICKHOUSE_DATABASE}" -db="${CLICKHOUSE_DATABASE}" - -${CLICKHOUSE_CLIENT} -q " - DROP USER IF EXISTS ${user}; - CREATE USER ${user}; - GRANT CREATE TABLE ON ${db}.* TO ${user}; - GRANT CLUSTER ON *.* TO ${user}; - GRANT TABLE ENGINE ON MergeTree TO ${user}; - CREATE TABLE ${db}.plain_src (id UInt64) ENGINE = MergeTree ORDER BY id; -" - -echo "without SHOW COLUMNS:" -${CLICKHOUSE_CLIENT} --user "${user}" --distributed_ddl_output_mode throw \ - --distributed_ddl_entry_format_version 2 \ - -q "CREATE TABLE ${db}.copy ON CLUSTER test_shard_localhost ENGINE = MergeTree ORDER BY id AS ${db}.plain_src" 2>&1 \ - | grep -oE "necessary to have the grant SHOW COLUMNS ON ${db}\.plain_src" | sed "s/${db}/db/" - -${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" From 2557c00dfd6b7765e29026d1f0e6fcd374ce5fda Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:06:13 +0000 Subject: [PATCH 44/81] Backport #126046 to 26.8: Do not read part dictionaries when estimating subcolumn sizes for PREWHERE --- .../MergeTree/MergeTreeReaderWide.cpp | 15 +++++- ...mn_size_without_reading_prefixes.reference | 3 ++ ...ubcolumn_size_without_reading_prefixes.sql | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.reference create mode 100644 tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 7480fb55a34e8..6f8b49239eba8 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -22,6 +22,14 @@ namespace DB namespace { constexpr auto DATA_FILE_EXTENSION = ".bin"; + + /// The streams of these columns are known only after reading the serialization prefixes: + /// the structure of dynamic types and the number of buckets of non-basic `Map`s are stored there. + bool streamsDependOnPrefixes(const NameAndTypePair & column, const SerializationInfoByName & infos) + { + return column.getTypeInStorage()->hasDynamicStructure() + || infos.getSettings().map_serialization_version != MergeTreeMapSerializationVersion::BASIC; + } } namespace ErrorCodes @@ -681,8 +689,11 @@ void MergeTreeReaderWide::readData( std::unordered_map> MergeTreeReaderWide::getAllColumnsSubstreams() { - /// We need to read prefixes to be able to collect all streams (because of dynamic structure of some columns). - deserializePrefixForAllColumns(columns_to_read.size(), 0); + /// Prefixes of other columns are not needed to list the streams and can be large, e.g. `LowCardinality` dictionaries. + const auto & infos = data_part_info_for_read->getSerializationInfos(); + if (std::ranges::any_of(columns_to_read, [&](const auto & column) { return streamsDependOnPrefixes(column, infos); })) + deserializePrefixForAllColumns(columns_to_read.size(), 0); + std::unordered_map> column_to_streams; for (size_t i = 0; i < columns_to_read.size(); ++i) { diff --git a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.reference b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.reference new file mode 100644 index 0000000000000..f5a2922a293c3 --- /dev/null +++ b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.reference @@ -0,0 +1,3 @@ +10 +1 1 +0 0 diff --git a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql new file mode 100644 index 0000000000000..573667b7263e1 --- /dev/null +++ b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql @@ -0,0 +1,47 @@ +-- Estimating the size of a `Map(LowCardinality(String), String)` key subcolumn for PREWHERE +-- must not read the dictionaries of the parts. + +SET allow_calculating_subcolumns_sizes_for_merge_tree_reading = 1; +SET optimize_functions_to_subcolumns = 1; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; +SET use_query_cache = 0; +-- Reading the statistics of the parts during planning is not what is measured. +SET use_statistics = 0; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_subcolumn_size_lc_map; + +CREATE TABLE t_subcolumn_size_lc_map (s String, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'basic'; + +SYSTEM STOP MERGES t_subcolumn_size_lc_map; + +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); +INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toString(number % 10), toString(number)) FROM numbers(1000); + +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_subcolumn_size_lc_map' AND active AND part_type = 'Wide'; + +SELECT countIf(explain ILIKE '%Prewhere filter%') > 0, countIf(explain ILIKE '%m.key_k1%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM t_subcolumn_size_lc_map WHERE m['k1'] = '1' AND s = '3') +SETTINGS log_comment = '05401_subcolumn_size_without_reading_prefixes_explain'; + +SYSTEM FLUSH LOGS query_log; + +SELECT ProfileEvents['ReadCompressedBytes'], ProfileEvents['LoadedMarksFiles'] +FROM system.query_log +WHERE current_database = currentDatabase() + AND log_comment = '05401_subcolumn_size_without_reading_prefixes_explain' + AND type = 'QueryFinish'; + +DROP TABLE t_subcolumn_size_lc_map; From 99606b3559db2b04f32893a36addad7da7e5031e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:43:18 +0000 Subject: [PATCH 45/81] Backport #125893 to 26.8: Fix a crash and lost rows on INSERT when a row policy of a table read by a materialized view uses `IN ` --- src/Planner/Utils.cpp | 5 +++ ...v_source_table_in_hidden_filters.reference | 5 +++ ...5402_mv_source_table_in_hidden_filters.sql | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference create mode 100644 tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 32c307f617736..4f6d2b152442c 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -711,6 +712,10 @@ QueryTreeNodePtr buildFilterQueryTree(ASTPtr filter_expression, if (check_access_rights) checkAccessRightsForFilter(filter_query_tree, table_expression, query_context); + /// The inner query of an ordinary view reads the table itself, not the inserted block. + if (auto view_source = query_context->getViewSource(); view_source && !query_context->isViewInnerQuery()) + replaceStorageInQueryTree(filter_query_tree, query_context, view_source); + return filter_query_tree; } diff --git a/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference new file mode 100644 index 0000000000000..a71fde4e65b88 --- /dev/null +++ b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference @@ -0,0 +1,5 @@ +additional_table_filters [2,3] +null_source [2,3] +policy_in_join [2,3] +policy_in_table [2,3] +view_inner [1] diff --git a/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql new file mode 100644 index 0000000000000..5ea7a7dbf9460 --- /dev/null +++ b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql @@ -0,0 +1,43 @@ +-- A row policy or `additional_table_filters` of a table read by a materialized view that names the view's source table +-- reads the inserted block, like the rest of the view's query. + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO src VALUES (1); +CREATE TABLE src_null (x UInt64) ENGINE = Null; + +CREATE TABLE child_policy (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_join (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_atf (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_null (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_view (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE other (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO child_policy VALUES (1), (2), (3); +INSERT INTO child_join VALUES (1), (2), (3); +INSERT INTO child_atf VALUES (1), (2), (3); +INSERT INTO child_null VALUES (1), (2), (3); +INSERT INTO child_view VALUES (1); +INSERT INTO other VALUES (1), (2), (3), (4); + +CREATE ROW POLICY OR REPLACE p_in_table ON child_policy USING x IN src TO ALL; +CREATE ROW POLICY OR REPLACE p_in_join ON child_join USING x IN (SELECT o.x FROM other AS o INNER JOIN src AS s ON o.x = s.x) TO ALL; +CREATE ROW POLICY OR REPLACE p_null ON child_null USING x IN src_null TO ALL; +CREATE ROW POLICY OR REPLACE p_view ON child_view USING x IN src TO ALL; +CREATE VIEW v AS SELECT x FROM child_view; + +CREATE TABLE dst (name String, x UInt64) ENGINE = MergeTree ORDER BY (name, x); +CREATE MATERIALIZED VIEW mv_in_table TO dst AS SELECT 'policy_in_table' AS name, c.x AS x FROM src AS s CROSS JOIN child_policy AS c; +CREATE MATERIALIZED VIEW mv_in_join TO dst AS SELECT 'policy_in_join' AS name, c.x AS x FROM src AS s CROSS JOIN child_join AS c; +CREATE MATERIALIZED VIEW mv_atf TO dst AS SELECT 'additional_table_filters' AS name, c.x AS x FROM src AS s CROSS JOIN child_atf AS c; +CREATE MATERIALIZED VIEW mv_view TO dst AS SELECT 'view_inner' AS name, c.x AS x FROM src AS s CROSS JOIN v AS c; +CREATE MATERIALIZED VIEW mv_null TO dst AS SELECT 'null_source' AS name, c.x AS x FROM src_null AS s CROSS JOIN child_null AS c; + +-- Not an asynchronous insert: its flush context has no current database. +INSERT INTO src SETTINGS async_insert = 0, additional_table_filters = {'child_atf': 'x IN src'} VALUES (2), (3), (4); +INSERT INTO src_null SETTINGS async_insert = 0 VALUES (2), (3), (4); + +SELECT name, groupArray(x) FROM (SELECT DISTINCT name, x FROM dst ORDER BY name, x) GROUP BY name ORDER BY name; + +DROP ROW POLICY p_in_table ON child_policy; +DROP ROW POLICY p_in_join ON child_join; +DROP ROW POLICY p_null ON child_null; +DROP ROW POLICY p_view ON child_view; From d85111bf9f3843ada9abff213e272645c89fffdc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 15:25:40 +0000 Subject: [PATCH 46/81] Update autogenerated version to 26.8.31.2 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 1729c22bbb464..67bf2f388cb48 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54543) +SET(VERSION_REVISION 54544) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 31) -SET(VERSION_GITHASH da45179b6c6761981c969859d168dd93b2bfd550) -SET(VERSION_DESCRIBE v26.8.31.1-lts) -SET(VERSION_STRING 26.8.31.1) +SET(VERSION_PATCH 32) +SET(VERSION_GITHASH bb452427e21b3693aef652d597bb7fe4437b088a) +SET(VERSION_DESCRIBE v26.8.32.1-lts) +SET(VERSION_STRING 26.8.32.1) # end of autochange From b2df76b0a78c8a4157ccdb1f9bb53f1853bf0277 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 18:11:31 +0000 Subject: [PATCH 47/81] Backport #126388 to 26.8: Don't keep a copy of the disk's `DataSourceDescription` in every backup entry --- src/Backups/BackupEntryFromImmutableFile.cpp | 8 ++++++-- src/Backups/BackupEntryFromImmutableFile.h | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Backups/BackupEntryFromImmutableFile.cpp b/src/Backups/BackupEntryFromImmutableFile.cpp index 5b481e611baf0..6c2aac2179531 100644 --- a/src/Backups/BackupEntryFromImmutableFile.cpp +++ b/src/Backups/BackupEntryFromImmutableFile.cpp @@ -15,8 +15,7 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile( bool allow_checksum_from_remote_path_) : disk(disk_) , file_path(file_path_) - , data_source_description(disk->getDataSourceDescription()) - , copy_encrypted(copy_encrypted_ && data_source_description.is_encrypted) + , copy_encrypted(copy_encrypted_ && disk_->getDataSourceDescription().is_encrypted) , passed_file_size(file_size_) , passed_checksum(checksum_) , allow_checksum_from_remote_path(allow_checksum_from_remote_path_) @@ -25,6 +24,11 @@ BackupEntryFromImmutableFile::BackupEntryFromImmutableFile( BackupEntryFromImmutableFile::~BackupEntryFromImmutableFile() = default; +DataSourceDescription BackupEntryFromImmutableFile::getDataSourceDescription() const +{ + return disk->getDataSourceDescription(); +} + std::unique_ptr BackupEntryFromImmutableFile::getReadBuffer(const ReadSettings & read_settings) const { if (copy_encrypted) diff --git a/src/Backups/BackupEntryFromImmutableFile.h b/src/Backups/BackupEntryFromImmutableFile.h index d286bc1ea21d2..bacd89e3604c6 100644 --- a/src/Backups/BackupEntryFromImmutableFile.h +++ b/src/Backups/BackupEntryFromImmutableFile.h @@ -28,7 +28,9 @@ class BackupEntryFromImmutableFile : public BackupEntryWithChecksumCalculation std::unique_ptr getReadBuffer(const ReadSettings & read_settings) const override; UInt64 getSize() const override; - DataSourceDescription getDataSourceDescription() const override { return data_source_description; } + /// Asked from the disk rather than kept in the entry: a backup has one entry per file of every part, so for a + /// large table the copies hold hundreds of megabytes describing the same few disks. + DataSourceDescription getDataSourceDescription() const override; bool isEncryptedByDisk() const override { return copy_encrypted; } bool isFromFile() const override { return true; } bool isFromImmutableFile() const override { return true; } @@ -45,7 +47,6 @@ class BackupEntryFromImmutableFile : public BackupEntryWithChecksumCalculation const DiskPtr disk; const String file_path; - const DataSourceDescription data_source_description; const bool copy_encrypted; const std::optional passed_file_size; const std::optional passed_checksum; From bafd3dda84cf7528cba9b0de9a0dbdd9461b25ca Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 20:50:27 +0000 Subject: [PATCH 48/81] Backport #126550 to 26.8: Limit nesting depth of messages in Protobuf schemas --- src/Common/checkStackSize.cpp | 10 +-- src/Formats/ProtobufSerializer.cpp | 42 ++++++++++- ...5422_protobuf_deep_message_chain.reference | 5 ++ .../05422_protobuf_deep_message_chain.sh | 72 +++++++++++++++++++ ...423_stack_size_too_large_message.reference | 2 + .../05423_stack_size_too_large_message.sh | 19 +++++ 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference create mode 100755 tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh create mode 100644 tests/queries/0_stateless/05423_stack_size_too_large_message.reference create mode 100755 tests/queries/0_stateless/05423_stack_size_too_large_message.sh diff --git a/src/Common/checkStackSize.cpp b/src/Common/checkStackSize.cpp index 8f40ac93c7294..89dfcc883db32 100644 --- a/src/Common/checkStackSize.cpp +++ b/src/Common/checkStackSize.cpp @@ -123,11 +123,13 @@ static NO_INLINE size_t getStackSize(void ** out_address) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Frame address is greater than stack begin address"); } -[[noreturn]] [[gnu::cold]] NO_INLINE static void throwTooDeepRecursion(const void * stack_begin, const void * frame, size_t used_stack_size, size_t total_stack_size) +/// The message is sent to the client, so it does not contain the addresses of the stack and the frame, +/// to not disclose the layout of the address space of the server. +[[noreturn]] [[gnu::cold]] NO_INLINE static void throwTooDeepRecursion(size_t used_stack_size, size_t total_stack_size) { throw DB::Exception(DB::ErrorCodes::TOO_DEEP_RECURSION, - "Stack size too large. Stack address: {}, frame address: {}, stack size: {}, maximum stack size: {}", - stack_begin, frame, used_stack_size, total_stack_size); + "Stack size too large. Stack size: {}, maximum stack size: {}", + used_stack_size, total_stack_size); } void checkStackSize() @@ -165,5 +167,5 @@ void checkStackSize() /// Just check if we have eat more than a STACK_SIZE_FREE_RATIO of stack size already. if (unlikely(stack_size > max_stack_size_allowed)) - throwTooDeepRecursion(stack_bounds.address, frame_address, stack_size, stack_bounds.max_size); + throwTooDeepRecursion(stack_size, stack_bounds.max_size); } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 1a11e40e8cf40..b9f13b1ef6f29 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -37,8 +37,10 @@ # include # include # include +# include # include # include +# include # include # include @@ -65,6 +67,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int ILLEGAL_COLUMN; + extern const int TOO_DEEP_RECURSION; } namespace @@ -74,6 +77,11 @@ namespace using FieldTypeId = google::protobuf::FieldDescriptor::Type; using OneofDescriptor = google::protobuf::OneofDescriptor; + /// The maximum nesting depth of messages we process. The schema is provided by the user, + /// and a chain of distinct messages, each having a field of the type of the next one, is not limited + /// by the protobuf library, while the functions below process it recursively. + constexpr size_t MAX_PROTOBUF_MESSAGE_NESTING_DEPTH = 1000; + /// Compares column's name with protobuf field's name. /// This comparison is case-insensitive and ignores the difference between '.' and '_' @@ -3456,6 +3464,18 @@ namespace bool columns_are_reordered_outside, bool check_nested_while_filling_missing_columns) { + checkStackSize(); + + if (message_nesting_depth >= MAX_PROTOBUF_MESSAGE_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Protobuf messages are nested too deeply (more than {} levels) at the message {}", + MAX_PROTOBUF_MESSAGE_NESTING_DEPTH, + quoteString(message_descriptor.full_name())); + + ++message_nesting_depth; + SCOPE_EXIT({ --message_nesting_depth; }); + std::vector field_descs; boost::container::flat_map field_descriptors_in_use; @@ -4159,6 +4179,9 @@ namespace const ProtobufReaderOrWriter reader_or_writer; std::function get_root_desc_function; std::shared_ptr root_serializer_ptr; + + /// The current depth of recursion of `buildMessageSerializerImpl`. + size_t message_nesting_depth = 0; }; template @@ -4182,7 +4205,18 @@ namespace bool allow_repeat, std::unordered_set & unresolved_descriptors) { + checkStackSize(); + chassert(unresolved_descriptors.contains(field_descriptor)); + + /// `unresolved_descriptors` contains the chain of fields of nested messages leading to the current field. + /// It protects only from recursive types, but not from long chains of distinct messages. + if (unresolved_descriptors.size() > MAX_PROTOBUF_MESSAGE_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Protobuf messages are nested too deeply (more than {} levels) at the field {}", + MAX_PROTOBUF_MESSAGE_NESTING_DEPTH, + quoteString(field_descriptor->full_name())); if (allow_repeat && field_descriptor->is_map()) { /// We don't add the same unresolved descriptor again since we are trying to re-resolve and put in under a Tuple @@ -4273,11 +4307,14 @@ namespace nested_field_descriptor->full_name()); } + /// Remove the descriptor on every exit, including skipping an unsupported field, + /// so that `unresolved_descriptors` contains only the chain of fields leading to the current one. + SCOPE_EXIT({ unresolved_descriptors.erase(nested_field_descriptor); }); + auto nested_name_and_type = getNameAndDataTypeFromFieldRecursive( nested_field_descriptor, skip_unsupported_fields, true, unresolved_descriptors); if (!nested_name_and_type) return std::nullopt; - unresolved_descriptors.erase(nested_field_descriptor); return NameAndTypePair{fmt::format("{}_{}", field_descriptor->name(), nested_name_and_type->name), nested_name_and_type->type}; } @@ -4294,11 +4331,12 @@ namespace "ClickHouse doesn't support type recursion ({})", message_descriptor->field(i)->full_name()); } + SCOPE_EXIT({ unresolved_descriptors.erase(message_descriptor->field(i)); }); + auto nested_name_and_type = getNameAndDataTypeFromFieldRecursive( message_descriptor->field(i), skip_unsupported_fields, true, unresolved_descriptors); if (!nested_name_and_type) continue; - unresolved_descriptors.erase(message_descriptor->field(i)); nested_types.push_back(nested_name_and_type->type); nested_names.push_back(nested_name_and_type->name); } diff --git a/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference new file mode 100644 index 0000000000000..b5b0aff8142ad --- /dev/null +++ b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference @@ -0,0 +1,5 @@ +f_f_f_x UInt32 +OK +080a060a040a020801 +OK +m Tuple(\n x UInt32) diff --git a/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh new file mode 100755 index 0000000000000..4dc2a529222e2 --- /dev/null +++ b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A chain of distinct messages, each having a field of the type of the next one, is not limited +# by the protobuf library, but was processed recursively both by the schema inference and while +# building the serializer, so a long chain used to exhaust the thread stack. + +# Prints a schema with the messages `M0`, ..., `M`, where `M` has the field `f` of the type `M`, +# and the last one has the field `x` of the type `uint32`. +make_schema() { + python3 -c " +depth = $1 +print('syntax = \"proto3\"; ' + ' '.join('message M{}{{M{} f=1;}}'.format(i, i + 1) for i in range(depth)) + ' message M{}{{uint32 x=1;}}'.format(depth)) +" +} + +# Prints the name of a column matching the field `x` of the last message, like `f.f.f.x`. +make_column_name() { + python3 -c "print('f.' * $1 + 'x')" +} + +check_too_deep() { + local output + output=$(cat) + if grep -q -F 'TOO_DEEP_RECURSION' <<< "$output" + then + echo "OK" + else + echo "FAIL: $(head -c 1000 <<< "$output" | tr '\n' ' ')" + fi +} + +run_desc() { + ${CLICKHOUSE_LOCAL} --query " + DESC format(Protobuf, '') + SETTINGS format_schema_source = 'string', format_schema = '$(make_schema "$1")', format_schema_message_name = 'M0' + " +} + +run_write() { + ${CLICKHOUSE_LOCAL} --query " + SELECT toUInt32(1) AS \`$(make_column_name "$1")\` + FORMAT Protobuf + SETTINGS format_schema_source = 'string', format_schema = '$(make_schema "$1")', format_schema_message_name = 'M0' + " +} + +DEPTH=2000 + +# Schema inference. +run_desc 3 +run_desc $DEPTH 2>&1 | check_too_deep + +# Building the serializer for a column nested as deep as the chain. +run_write 3 | od -An -tx1 -v | tr -d ' \n' +echo +run_write $DEPTH 2>&1 | check_too_deep + +# Unsupported fields skipped during schema inference must not count as nesting levels: +# a shallow message with many skipped fields of an empty message type. +SHALLOW_SCHEMA=$(python3 -c " +print('syntax = \"proto3\"; message E {} message Inner {' + ' '.join('E e{}={};'.format(i, i + 1) for i in range(1500)) + ' uint32 x=1501;} message Root {Inner m=1;}') +") +${CLICKHOUSE_LOCAL} --query " +DESC format(Protobuf, '') +SETTINGS format_schema_source = 'string', format_schema = '$SHALLOW_SCHEMA', format_schema_message_name = 'Root', + input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference = 1 +" diff --git a/tests/queries/0_stateless/05423_stack_size_too_large_message.reference b/tests/queries/0_stateless/05423_stack_size_too_large_message.reference new file mode 100644 index 0000000000000..b261da18d51a3 --- /dev/null +++ b/tests/queries/0_stateless/05423_stack_size_too_large_message.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/05423_stack_size_too_large_message.sh b/tests/queries/0_stateless/05423_stack_size_too_large_message.sh new file mode 100755 index 0000000000000..4837cd31cfeb7 --- /dev/null +++ b/tests/queries/0_stateless/05423_stack_size_too_large_message.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The message of the `TOO_DEEP_RECURSION` exception thrown by `checkStackSize` is sent to the client, +# so it must not contain the addresses of the stack and the frame of the server. +# Two `Merge` tables reading from each other recurse until the stack check fires. + +output=$(${CLICKHOUSE_LOCAL} --query " +CREATE TABLE merge1 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\\\d\$'); +CREATE TABLE merge2 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\\\d\$'); +SELECT * FROM merge1; +" 2>&1) + +# `grep -c` exits with code 1 when the count is zero, so do not let it fail the test. +grep -c -F 'Stack size too large' <<< "$output" ||: +grep -c -F 'address' <<< "$output" ||: From 771b196358383bdebdf01f641df0c55faa0c74bc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 21:30:23 +0000 Subject: [PATCH 49/81] Backport #126547 to 26.8: Limit nesting depth of arrays in Redis replies --- base/poco/Redis/include/Poco/Redis/Array.h | 24 +++++++ .../Redis/include/Poco/Redis/RedisStream.h | 13 ++++ base/poco/Redis/src/Client.cpp | 15 +++- ..._redis_deeply_nested_array_reply.reference | 3 + .../05426_redis_deeply_nested_array_reply.sh | 72 +++++++++++++++++++ 5 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference create mode 100755 tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh diff --git a/base/poco/Redis/include/Poco/Redis/Array.h b/base/poco/Redis/include/Poco/Redis/Array.h index 2849829638d73..2f6a3dcf442d7 100644 --- a/base/poco/Redis/include/Poco/Redis/Array.h +++ b/base/poco/Redis/include/Poco/Redis/Array.h @@ -301,10 +301,34 @@ namespace Redis return result.str(); } + /// The maximum allowed nesting depth of arrays in a reply. + /// Replies of real Redis servers are nested only a few levels deep, + /// while unbounded nesting would lead to stack overflow in the recursive parser. + static constexpr int MAX_NESTING_DEPTH = 128; + static void read(RedisInputStream & input, Array & value) { value.clear(); + struct DepthGuard + { + explicit DepthGuard(int & depth_) : depth(depth_) + { + ++depth; + } + + ~DepthGuard() + { + --depth; + } + + int & depth; + }; + + DepthGuard depth_guard(input.arrayNestingDepth()); + if (depth_guard.depth > MAX_NESTING_DEPTH) + throw RedisException("Too deep nesting of arrays in the answer received from Redis server"); + Int64 length = NumberParser::parse64(input.getline()); if (length != -1) diff --git a/base/poco/Redis/include/Poco/Redis/RedisStream.h b/base/poco/Redis/include/Poco/Redis/RedisStream.h index 128efcfb141fa..29183988871dd 100644 --- a/base/poco/Redis/include/Poco/Redis/RedisStream.h +++ b/base/poco/Redis/include/Poco/Redis/RedisStream.h @@ -107,9 +107,22 @@ namespace Redis std::string getline(); /// Redis uses \r\n as delimiter. This getline version removes /// the \r from the result. + + int & arrayNestingDepth(); + /// Returns the current nesting depth of arrays being read from the stream. + /// Used to limit the recursion depth while parsing nested arrays. + + private: + int _arrayNestingDepth = 0; }; + inline int & RedisInputStream::arrayNestingDepth() + { + return _arrayNestingDepth; + } + + } } // namespace Poco::Redis diff --git a/base/poco/Redis/src/Client.cpp b/base/poco/Redis/src/Client.cpp index 6d28f6a29d26f..99a4ea9094bfc 100644 --- a/base/poco/Redis/src/Client.cpp +++ b/base/poco/Redis/src/Client.cpp @@ -175,10 +175,23 @@ RedisType::Ptr Client::readReply() RedisType::Ptr result = RedisType::createRedisType(c); if (result.isNull()) { + /// The stream is not at a reply boundary anymore, so the connection cannot be reused. + disconnect(); throw RedisException("Invalid Redis type returned"); } - result->read(*_input); + try + { + result->read(*_input); + } + catch (...) + { + /// A partially read reply leaves unread data in the stream. + /// If the connection were reused (e.g. returned to a pool), the next request + /// would read the remainder of this reply instead of its own one. + disconnect(); + throw; + } return result; } diff --git a/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference new file mode 100644 index 0000000000000..9591613d44cf9 --- /dev/null +++ b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference @@ -0,0 +1,3 @@ +OK +OK +0 diff --git a/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh new file mode 100755 index 0000000000000..3ab3dc2af914b --- /dev/null +++ b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A fake Redis server replies to a command mentioning the key 'deep' with an array nested 100000 levels deep, +# and to any other command with an array of nils (one per argument except the command name). +# Parsing the deep reply must fail with an exception instead of a stack overflow, +# and the connection must not be reused: the next query over the same single-connection pool +# must receive its own reply instead of the unread remainder of the deep one. + +PORT_FILE="${CLICKHOUSE_TMP}/redis_port" +rm -f "$PORT_FILE" + +python3 -c " +import os, socket, sys, threading +deep_reply = b'*1\r\n' * 100000 + b':1\r\n' +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(('127.0.0.1', 0)) +server.listen() +server.settimeout(60) +with open(sys.argv[1] + '.tmp', 'w') as f: + f.write(str(server.getsockname()[1])) +os.rename(sys.argv[1] + '.tmp', sys.argv[1]) + +def serve(conn): + reader = conn.makefile('rb') + with conn: + try: + while True: + header = reader.readline() + if not header: + return + num_args = int(header[1:]) + args = [] + for _ in range(num_args): + length = int(reader.readline()[1:]) + args.append(reader.read(length + 2)[:-2]) + if any(b'deep' in arg for arg in args): + conn.sendall(deep_reply) + else: + conn.sendall(b'*%d\r\n' % (num_args - 1) + b'\$-1\r\n' * (num_args - 1)) + except (OSError, ValueError): + pass + +try: + while True: + conn, _ = server.accept() + threading.Thread(target=serve, args=(conn,), daemon=True).start() +except socket.timeout: + pass +" "$PORT_FILE" & +SERVER_PID=$! + +for _ in {1..100}; do + [ -f "$PORT_FILE" ] && break + sleep 0.1 +done +PORT=$(cat "$PORT_FILE") + +$CLICKHOUSE_CLIENT --query "SELECT * FROM redis('127.0.0.1:${PORT}', 'k', 'k String, v String') WHERE k = 'deep'" 2>&1 | grep -q -F 'Too deep nesting of arrays' && echo OK + +$CLICKHOUSE_CLIENT --query "CREATE TABLE redis_deep (k String, v String) ENGINE = Redis('127.0.0.1:${PORT}', 0, '', 1) PRIMARY KEY k" +$CLICKHOUSE_CLIENT --query "SELECT * FROM redis_deep WHERE k = 'deep'" 2>&1 | grep -q -F 'Too deep nesting of arrays' && echo OK +$CLICKHOUSE_CLIENT --query "SELECT count() FROM redis_deep WHERE k = 'a'" +$CLICKHOUSE_CLIENT --query "DROP TABLE redis_deep" + +kill $SERVER_PID 2>/dev/null +wait $SERVER_PID 2>/dev/null +rm -f "$PORT_FILE" From d46de11c024273e22eb7cc7a3e221fce25b44a4b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 21:40:23 +0000 Subject: [PATCH 50/81] Backport #126159 to 26.8: Fix segfault in `MySQLDump` format on `ENUM` without values --- .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 8 +++++++- ...mysqldump_enum_without_arguments.reference | 7 +++++++ .../05401_mysqldump_enum_without_arguments.sh | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference create mode 100755 tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index 9c639c653fb0c..0c35e80bc0910 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -223,12 +223,18 @@ static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_defini /// Note: Enum would be transformed to Enum8 if number of elements is less then 128, otherwise it would be transformed to Enum16. if (type_name_upper.contains("ENUM")) { + ASTPtr arguments = data_type_node->getArguments(); + if (!arguments) + throw Exception(ErrorCodes::INCORRECT_DATA, "Type {} in MySQL dump requires a list of values", data_type_node->name); + UInt16 i = 0; - for (ASTPtr & child : data_type_node->getArguments()->children) + for (ASTPtr & child : arguments->children) { auto new_child = make_intrusive(); new_child->name = "equals"; auto * literal = child->as(); + if (!literal || literal->value.getType() != Field::Types::String) + throw Exception(ErrorCodes::INCORRECT_DATA, "Values of type {} in MySQL dump must be string literals", data_type_node->name); new_child->arguments = make_intrusive(); new_child->arguments->children.push_back(make_intrusive(literal->value.safeGet())); diff --git a/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference new file mode 100644 index 0000000000000..600e544c7f992 --- /dev/null +++ b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference @@ -0,0 +1,7 @@ +ENUM +INCORRECT_DATA +ENUM(UInt8) +INCORRECT_DATA +ENUM(1, 2) +INCORRECT_DATA +b Enum8(\'a\' = 1, \'b\' = 2) diff --git a/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh new file mode 100755 index 0000000000000..d8b53b38873bb --- /dev/null +++ b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A MySQL dump with `ENUM` without the list of values, or with non-string values, must be rejected with an exception. + +function dump() +{ + printf 'CREATE TABLE `t1` (`x` %s NOT NULL) ENGINE=InnoDB;\n/*!40101 SET character_set_client = @saved_cs_client */;\nINSERT INTO `t1` VALUES (%s);\n' "$1" "$2" +} + +for type in 'ENUM' 'ENUM(UInt8)' 'ENUM(1, 2)'; do + echo "$type" + dump "$type" 1 | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" 2>&1 | grep -o -m1 -F 'INCORRECT_DATA' +done + +dump "ENUM('a', 'b')" "'b'" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT x, toTypeName(x) FROM table" From e37b5dbdf27f0668b4e798242ac6274ecb39a26f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 22:48:36 +0000 Subject: [PATCH 51/81] Backport #120776 to 26.8: Fix uninitialized offsets in Variant/Dynamic/JSON columns containing only NULLs --- src/Columns/ColumnVariant.cpp | 3 +- ...offsets_uninitialized_only_nulls.reference | 4 ++ ...riant_offsets_uninitialized_only_nulls.sql | 41 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference create mode 100644 tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 7ce1c53ff4b03..217675fea173f 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -75,7 +75,8 @@ void ColumnVariant::constructOffsetsFromDiscriminators() /// If we have only NULLs, offsets column will not contain any real offsets. if (hasOnlyNulls()) { - offsets_data.resize(discriminators_concrete->size()); + /// Offsets of NULL rows are never read for their value, but compression reads the whole buffer. + offsets_data.resize_fill(discriminators_concrete->size(), 0); } /// If we have only one non empty variant and no NULLs, /// offsets column will contain just sequential offsets 0, 1, 2, ... diff --git a/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference new file mode 100644 index 0000000000000..4cba3ca86c550 --- /dev/null +++ b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference @@ -0,0 +1,4 @@ +2048 2048 +1 +2048 2048 +1 diff --git a/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql new file mode 100644 index 0000000000000..8f8092aa585bc --- /dev/null +++ b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql @@ -0,0 +1,41 @@ +-- Tags: memory-engine +-- Offsets of NULL rows in a Variant column are meaningless, but ColumnVariant::compress hands the +-- whole offsets buffer to LZ4, which reads every byte. Only a sanitizer build can observe this. +-- Sizing is load bearing: ColumnVector::compress skips buffers below 4096 bytes, so the block that +-- reaches the compressing sink must keep at least 512 rows. The insert block sizes are pinned so +-- that neither randomized settings nor squashing can split or re-create the block. + +DROP TABLE IF EXISTS t_variant_src; +DROP TABLE IF EXISTS t_variant_dst; +DROP TABLE IF EXISTS t_json_src; +DROP TABLE IF EXISTS t_json_dst; + +CREATE TABLE t_variant_src (n UInt64, v Variant(UInt64, String)) ENGINE = Memory; +INSERT INTO t_variant_src SELECT number, if(number % 2 = 0, NULL, number)::Variant(UInt64, String) FROM numbers(4096) +SETTINGS max_block_size = 4096, min_insert_block_size_rows = 4096, min_insert_block_size_bytes = 0; + +CREATE TABLE t_variant_dst (v Variant(UInt64, String)) ENGINE = Memory SETTINGS compress = 1; +INSERT INTO t_variant_dst SELECT v FROM t_variant_src WHERE n % 2 = 0 +SETTINGS min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; + +SELECT count(), countIf(v IS NULL) FROM t_variant_dst; + +-- The stored table must stay far below its raw offsets buffer of total_rows * 8 bytes. If this +-- line reddens, the block no longer reaches the compressing sink and the MSan oracle above is dead. +SELECT total_bytes < total_rows * 8 FROM system.tables WHERE database = currentDatabase() AND name = 't_variant_dst'; + +CREATE TABLE t_json_src (n UInt64, j JSON) ENGINE = Memory; +INSERT INTO t_json_src SELECT number, ('{"a":' || if(number % 2 = 0, 'null', toString(number)) || '}')::JSON FROM numbers(4096) +SETTINGS max_block_size = 4096, min_insert_block_size_rows = 4096, min_insert_block_size_bytes = 0; + +CREATE TABLE t_json_dst (j JSON) ENGINE = Memory SETTINGS compress = 1; +INSERT INTO t_json_dst SELECT j FROM t_json_src WHERE n % 2 = 0 +SETTINGS min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; + +SELECT count(), countIf(j.a IS NULL) FROM t_json_dst; +SELECT total_bytes < total_rows * 8 FROM system.tables WHERE database = currentDatabase() AND name = 't_json_dst'; + +DROP TABLE t_variant_src; +DROP TABLE t_variant_dst; +DROP TABLE t_json_src; +DROP TABLE t_json_dst; From 2c4c7289e88336ff8bb8cdbee5e9b064d9817569 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 08:35:26 +0000 Subject: [PATCH 52/81] Backport #126126 to 26.8: Fix segfault in adaptive aggregation after a failed drain into the shared table --- src/Common/FailPoint.cpp | 1 + src/Interpreters/AdaptiveAggregationImpl.h | 6 ++ .../AdaptiveAggregationKernels.cpp | 62 ++++++++++++++----- src/Interpreters/Aggregator.h | 9 +++ .../Transforms/AggregatingTransform.cpp | 7 +++ ...e_aggregator_broken_shared_drain.reference | 1 + ...adaptive_aggregator_broken_shared_drain.sh | 40 ++++++++++++ 7 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.reference create mode 100755 tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index a372a97e8b9c2..abeebe2da0a6f 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -196,6 +196,7 @@ static struct InitFiu REGULAR(refresh_mv_skip_execution) \ ONCE(column_aggregate_function_ensureOwnership_exception) \ ONCE(space_saving_copy_arena_throw) \ + ONCE(adaptive_aggregation_drain_create_states_throw) \ REGULAR(keepermap_fail_drop_data) \ PAUSEABLE_ONCE(keepermap_create_pause_before_drop_lock_version) \ REGULAR(keeper_fault_on_watch_request) \ diff --git a/src/Interpreters/AdaptiveAggregationImpl.h b/src/Interpreters/AdaptiveAggregationImpl.h index 782c921ba9500..75f3c7d8bf680 100644 --- a/src/Interpreters/AdaptiveAggregationImpl.h +++ b/src/Interpreters/AdaptiveAggregationImpl.h @@ -383,6 +383,12 @@ struct AdaptiveAggregationSession /// spilling gigabytes to disk - before it stops. std::atomic cancelled{false}; + /// Set when a drain into `early_drain_variants` threw midway, leaving keys with null states + /// there. Unlike `cancelled`, which a successful query can also see (a `LIMIT` cancelling the + /// pipeline), this means the table must never be merged or written: the last finisher + /// checks it and fails instead of finalizing. + std::atomic shared_table_broken{false}; + /// Stops the sweeps at their next check and wakes any producer waiting for spill budget; /// the fence keeps the store from racing past a waiter that already checked the flag. void cancel() diff --git a/src/Interpreters/AdaptiveAggregationKernels.cpp b/src/Interpreters/AdaptiveAggregationKernels.cpp index 32dbd5da782e0..d232d0857a050 100644 --- a/src/Interpreters/AdaptiveAggregationKernels.cpp +++ b/src/Interpreters/AdaptiveAggregationKernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,12 @@ namespace ErrorCodes { extern const int UNKNOWN_AGGREGATED_DATA_VARIANT; extern const int LOGICAL_ERROR; + extern const int FAULT_INJECTED; +} + +namespace FailPoints +{ + extern const char adaptive_aggregation_drain_create_states_throw[]; } } @@ -1408,6 +1415,12 @@ void NO_INLINE Aggregator::drainAdaptiveBucketImpl( if (inserted) { it->getMapped() = nullptr; + /// Stands for the creation of the states failing, as on a memory limit: the key stays + /// in the table with a null state. + fiu_do_on(FailPoints::adaptive_aggregation_drain_create_states_throw, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injected fault while creating the states of a drained key"); + }); aggregate_data = bucket_arena->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); createAggregateStates(aggregate_data, use_compiled_functions); it->getMapped() = aggregate_data; @@ -1809,6 +1822,38 @@ size_t Aggregator::drainStagedBatch( return drained; } +size_t Aggregator::drainStagedBatchIntoSharedTable( + AdaptiveAggregationSession & shared, + const std::vector & chunks, + PaddedPODArray & places_scratch) const +{ + /// The table's account grows by what this drain was seen to allocate - read before the + /// batch is released, so its bytes do not come off the reading - which is how the heap + /// its states own outside the arenas reaches the part bound. + const Int64 tracked_before_drain = currentThreadTrackedMemory(); + size_t drained = 0; + try + { + drained = drainStagedBatch(*shared.early_drain_variants, chunks, shared.cancelled, places_scratch); + } + catch (...) + { + /// The table may now hold keys with null states. The query is failing with this + /// exception, but until the executor cancels the transforms, other producers keep + /// running and take the coordinator lock next: a later drain would add rows into the + /// null states, a residue release would write the table to disk and serialize them. + /// Cancelling the session here, under the coordinator lock, stops every later drain + /// into the shared table and every detach and write of it. The finalization checks + /// `shared_table_broken` too, as the last finisher may get there before it is cancelled. + shared.shared_table_broken.store(true, std::memory_order_release); + shared.cancel(); + throw; + } + shared.early_drain_tracked_bytes + += static_cast(std::max(currentThreadTrackedMemory() - tracked_before_drain, 0)); + return drained; +} + void Aggregator::spillDetachedAdaptiveTable(AdaptiveAggregationSession & shared, AggregatedDataVariants & table) const { if (shared.cancelled.load(std::memory_order_relaxed)) @@ -1929,13 +1974,7 @@ void Aggregator::drainStagedChunksAtFinish(AdaptiveAggregationSession & shared) const std::vector batch( std::make_move_iterator(chunks.begin() + begin), std::make_move_iterator(chunks.begin() + end)); - /// The table's account grows by what this drain was seen to allocate - read before the - /// batch is released, so its bytes do not come off the reading - which is how the heap - /// its states own outside the arenas reaches the part bound above. - const Int64 tracked_before_drain = currentThreadTrackedMemory(); - drained_records += drainStagedBatch(*shared.early_drain_variants, batch, shared.cancelled, places_scratch); - shared.early_drain_tracked_bytes - += static_cast(std::max(currentThreadTrackedMemory() - tracked_before_drain, 0)); + drained_records += drainStagedBatchIntoSharedTable(shared, batch, places_scratch); begin = end; } @@ -2020,14 +2059,7 @@ bool Aggregator::drainStagedChunksBatchUnderMemoryPressure( while (shared.early_drain_variants->aggregates_pools.size() < ADAPTIVE_AGGREGATION_NUM_BUCKETS) shared.early_drain_variants->aggregates_pools.push_back(std::make_shared()); - /// The shared table's account grows by what this drain was seen to allocate, read - /// before the batch is released so its bytes do not come off the reading: that is - /// how the heap its states own outside the arenas counts toward the detachment below. - const Int64 tracked_before_drain = currentThreadTrackedMemory(); - const size_t drained_records - = drainStagedBatch(*shared.early_drain_variants, batch, shared.cancelled, places_scratch); - shared.early_drain_tracked_bytes - += static_cast(std::max(currentThreadTrackedMemory() - tracked_before_drain, 0)); + const size_t drained_records = drainStagedBatchIntoSharedTable(shared, batch, places_scratch); batch.clear(); ProfileEvents::increment(ProfileEvents::AdaptiveAggregationPressureDrainedRecords, drained_records); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index f12c6acbe42cf..42f0787448bcb 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -297,6 +297,15 @@ class Aggregator final std::atomic & is_cancelled, PaddedPODArray & places_scratch) const; + /// `drainStagedBatch` into the session's shared table, called under `pressure_sweep_mutex`. + /// A drain that throws midway (a memory limit hit while creating states, say) leaves keys + /// in the shared table whose states are null; the session is cancelled before the lock is + /// released, so no other producer drains into that table or writes it. + size_t drainStagedBatchIntoSharedTable( + AdaptiveAggregationSession & shared, + const std::vector & chunks, + PaddedPODArray & places_scratch) const; + /// A fresh drain destination of the session's method type, with one arena per bucket. AggregatedDataVariantsPtr createAdaptiveDrainTable(AggregatedDataVariants::Type type) const; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 6a7c49764e0f7..5bac43fcd02b9 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -44,6 +44,7 @@ namespace ErrorCodes { extern const int UNKNOWN_AGGREGATED_DATA_VARIANT; extern const int LOGICAL_ERROR; + extern const int ABORTED; } ManyAggregatedData::~ManyAggregatedData() @@ -1364,6 +1365,12 @@ void AggregatingTransform::initGenerate() return; } + /// A drain that failed in another producer may have left keys with null states in the shared + /// table, and the executor may not have cancelled this transform yet. Nothing below may merge + /// or write that table. + if (adaptive_context && adaptive_context->session->shared_table_broken.load(std::memory_order_acquire)) + throw Exception(ErrorCodes::ABORTED, "Adaptive aggregation cannot finish: a drain into its shared table failed"); + adaptive_engaged = adaptive_context && adaptive_context->session->initialized.load(std::memory_order_acquire); if (adaptive_engaged) diff --git a/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.reference b/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.reference new file mode 100644 index 0000000000000..f6cab5f812c54 --- /dev/null +++ b/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.reference @@ -0,0 +1 @@ +FAULT_INJECTED diff --git a/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.sh b/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.sh new file mode 100755 index 0000000000000..c98e88fca7329 --- /dev/null +++ b/tests/queries/0_stateless/05401_adaptive_aggregator_broken_shared_drain.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A pressure sweep that throws while draining into the adaptive session's shared table, for example +# on a memory limit hit while creating the states of a new key, leaves that key in the table with a +# null state. The other producers kept running until the executor cancelled them, and the next one +# to take the coordinator lock dereferenced the null state: a drain adding rows into it, or a residue +# release writing the table to disk through the serialization of `topK`. The failed drain now +# cancels the session before it releases the coordinator lock, so the query fails with the drain's +# exception. +# +# The failpoint fails the creation of the states of the first key a drain inserts. With a threshold +# of 200 MB the sweeps stay in the tail regime, which drains into the shared table, while the other +# producers queue on the coordinator lock. The race is narrow - without the fix about one run in +# seventy crashed, and only with 64 threads - so this exercises the path rather than reproducing the +# crash reliably. The outer query reads the aggregates, so they are not pruned from the inner one. + +$CLICKHOUSE_LOCAL --query " +SYSTEM ENABLE FAILPOINT adaptive_aggregation_drain_create_states_throw; +SET max_threads = 16; +SET max_block_size = 8192; +SET enable_adaptive_aggregator = 1; +SET adaptive_aggregator_freeze_threshold = 1000; +SET adaptive_aggregator_freeze_threshold_bytes = 0; +SET group_by_two_level_threshold = 1000; +SET group_by_two_level_threshold_bytes = 1000000; +SET max_bytes_before_external_group_by = 200000000; +SET max_bytes_ratio_before_external_group_by = 0; +SET collect_hash_table_stats_during_aggregation = 0; + +SELECT count(), sum(length(t)), sum(u) FROM +( + SELECT toString(number % 200000) AS k, topK(1, 100)(toString(number)) AS t, uniqExact(number) AS u + FROM numbers_mt(5000000) + GROUP BY k +); +" 2>&1 | grep -o -m1 'FAULT_INJECTED' From d424b0fdb2b66c58d3f512d3a4c646829442763b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 09:16:11 +0000 Subject: [PATCH 53/81] Backport #126292 to 26.8: Fix heap-use-after-free in `TablesDependencyGraph` and reject a materialized view selecting from itself --- src/Databases/DDLDependencyVisitor.cpp | 8 +-- src/Databases/DDLLoadingDependencyVisitor.cpp | 3 +- src/Databases/TablesDependencyGraph.cpp | 5 ++ .../tests/gtest_tables_dependency_graph.cpp | 19 +++++++ src/Storages/StorageMaterializedView.cpp | 22 ++++++++ ...d_view_cannot_select_from_itself.reference | 2 + ...ialized_view_cannot_select_from_itself.sql | 51 +++++++++++++++++++ 7 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference create mode 100644 tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index bc1ac29143ff0..17727b792a360 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -319,9 +319,10 @@ namespace } else if (functionIsInOrGlobalInOperator(function.name)) { - /// x IN table_name. + /// x IN table_name. A literal, as in x IN ('db.table'), is a set of values, not a table name. /// We set evaluate=false here because we don't want to evaluate a subquery in "x IN subquery". - addQualifiedNameFromArgument(function, 1, /* evaluate= */ false); + if (function.arguments && function.arguments->children.size() > 1 && !function.arguments->children[1]->as()) + addQualifiedNameFromArgument(function, 1, /* evaluate= */ false); } else if (function.name == "dictionary") { @@ -437,7 +438,8 @@ namespace /// We're just searching for dependencies here, it's not safe to execute subqueries now. /// Use copy of the global_context and set current database, because expressions can contain currentDatabase() function. ContextMutablePtr global_context_copy = Context::createCopy(global_context); - global_context_copy->setCurrentDatabase(current_database); + if (!current_database.empty()) + global_context_copy->setCurrentDatabase(current_database); auto evaluated = evaluateConstantExpressionOrIdentifierAsLiteral(arg, global_context_copy); const auto * literal = evaluated->as(); if (!literal || (literal->value.getType() != Field::Types::String)) diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index 672e3c258bd90..8cc824880ba7e 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -86,7 +86,8 @@ ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToVisit(const ASTFunction if (!args || args->children.size() != 2) return -1; - if (args->children[1]->as()) + /// A literal, as in x IN ('db.table'), is a set of values, not a table name. + if (args->children[1]->as() || args->children[1]->as()) return -1; return 1; diff --git a/src/Databases/TablesDependencyGraph.cpp b/src/Databases/TablesDependencyGraph.cpp index f0f61f973385b..7ad5bc526695b 100644 --- a/src/Databases/TablesDependencyGraph.cpp +++ b/src/Databases/TablesDependencyGraph.cpp @@ -93,7 +93,12 @@ size_t TablesDependencyGraph::getNumberOfTables() const void TablesDependencyGraph::addDependency(const StorageID & table_id, const StorageID & dependency) { auto * table_node = addOrUpdateNode(table_id); + + /// Adding `dependency` removes `table_node` from the graph if it gives the name of `table_node` to another node. + NodeSharedPtr table_node_holder = *nodes.find(table_node); auto * dependency_node = addOrUpdateNode(dependency); + if (!nodes.contains(table_node)) + table_node = dependency_node; /// It has the name `table_node` had, so it's the node of `table_id` now. bool inserted = table_node->dependencies.insert(dependency_node).second; if (!inserted) diff --git a/src/Databases/tests/gtest_tables_dependency_graph.cpp b/src/Databases/tests/gtest_tables_dependency_graph.cpp index eaf1e7ad9217f..b114536350cf8 100644 --- a/src/Databases/tests/gtest_tables_dependency_graph.cpp +++ b/src/Databases/tests/gtest_tables_dependency_graph.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -79,3 +80,21 @@ TEST(TablesDependencyGraph, SplitByDependencyLevelMixedCycle) EXPECT_EQ(split[1][0].getTableName(), "leaf"); EXPECT_EQ(split.back().size(), 2u); } + +/// After `RENAME` of a materialized view that selects from itself, the view graph holds the view +/// as two nodes: the old name with the UUID, and the new name. Adding the dependency of the new name +/// on the UUID gives the new name to the first node and removes the second one, the node of `table_id`. +TEST(TablesDependencyGraph, AddDependencyWhenTableNodeIsRemoved) +{ + TablesDependencyGraph graph("test"); + + UUID uuid = UUIDHelpers::generateV4(); + graph.addDependency(StorageID{"db", "mv", uuid}, StorageID{"db", "mv2"}); + graph.addDependency(StorageID{"db", "mv2"}, StorageID{"db", "mv2", uuid}); + + EXPECT_EQ(graph.getNumberOfTables(), 1u); + auto dependencies = graph.getDependencies(StorageID{"db", "mv2", uuid}); + ASSERT_EQ(dependencies.size(), 1u); + EXPECT_EQ(dependencies[0].getFullTableName(), "db.mv2"); + EXPECT_EQ(dependencies[0].uuid, uuid); +} diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 69ee422181f50..cb772f006fc75 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #include #include #include @@ -114,6 +116,17 @@ namespace if (!target_table_columns.has(column.name)) throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "Column {} does not exist in the materialized view's inner table", column.name); } + + bool selectReadsView(const StorageID & view_id, const ASTPtr & select_with_union, const ContextPtr & context) + { + /// Qualified as outside this session, where the name of a temporary table is a table of the view's database. + /// No current database: a recursive CTE stays unqualified. No `table_name`: it would be dropped from the result. + ASTPtr select_copy = select_with_union->clone(); + AddDefaultDatabaseVisitor(context->getGlobalContext(), view_id.database_name).visit(select_copy); + auto referenced_tables = getDependenciesFromCreateQuery( + context, /*table_name=*/{}, select_copy, /*current_database=*/"").dependencies; + return referenced_tables.contains(view_id.getQualifiedName()); + } } StorageMaterializedView::StorageMaterializedView( @@ -163,6 +176,8 @@ StorageMaterializedView::StorageMaterializedView( mv_db_context->setCurrentDatabase(table_id_.database_name); auto select = SelectQueryDescription::getSelectQueryFromASTForMatView(query.select->clone(), query.refresh_strategy != nullptr, mv_db_context); + if (mode == LoadingStrictnessLevel::CREATE && !query.refresh_strategy && selectReadsView(table_id_, select.select_query, mv_db_context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Materialized view {} cannot select from itself", table_id_.getFullTableName()); if (select.select_table_id) { auto select_table_dependent_views = DatabaseCatalog::instance().getDependentViews(select.select_table_id); @@ -802,6 +817,13 @@ void StorageMaterializedView::alter( params.apply(new_metadata, mv_db_context); const auto & new_select = new_metadata.select; + bool modifies_query = std::ranges::any_of(params, [](const AlterCommand & command) { return command.type == AlterCommand::MODIFY_QUERY; }); + /// A refreshable view is not triggered by inserts, so its query may read the view itself. + if (modifies_query && !new_metadata.refresh) + { + if (selectReadsView(table_id, new_select.select_query, mv_db_context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Materialized view {} cannot select from itself", table_id.getFullTableName()); + } new_metadata.setSelectQuery(new_select); /// Check the materialized view's inner table structure. diff --git a/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference new file mode 100644 index 0000000000000..ad05fc3309035 --- /dev/null +++ b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference @@ -0,0 +1,2 @@ +['mv'] +['mv2'] diff --git a/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql new file mode 100644 index 0000000000000..bc6867f0674d9 --- /dev/null +++ b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql @@ -0,0 +1,51 @@ +CREATE TABLE src (c0 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY tuple() AS SELECT c0 FROM src; +CREATE VIEW v AS SELECT c0 FROM mv; + +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM (SELECT c0 FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM (SELECT c0 FROM src JOIN mv USING (c0)); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src JOIN mv USING (c0); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src JOIN (SELECT c0 FROM mv) AS t USING (c0); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 > (SELECT count() FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src UNION ALL SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM cluster('test_shard_localhost', currentDatabase(), 'mv')); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM v; -- { serverError INFINITE_LOOP } +-- A string literal on the right side of IN is a set of values, not a table. +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE toString(c0) IN ('v'); +-- A recursive CTE named like the view is not the view. +ALTER TABLE mv MODIFY QUERY WITH RECURSIVE mv AS (SELECT 1 AS c0 UNION ALL SELECT c0 + 1 FROM mv WHERE c0 < 3) SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); +SELECT dependencies_table FROM system.tables WHERE database = currentDatabase() AND name = 'src'; +-- A temporary table named like the view is stored unqualified, so outside this session it is the view. +CREATE TEMPORARY TABLE mv (c0 Int32); +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv MODIFY QUERY SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv MODIFY QUERY WITH RECURSIVE mv AS (SELECT 1 AS c0 UNION ALL SELECT c0 + 1 FROM mv WHERE c0 < 3) SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); +DROP TEMPORARY TABLE mv; +CREATE TEMPORARY TABLE mv3 (c0 Int32); +CREATE MATERIALIZED VIEW {CLICKHOUSE_DATABASE:Identifier}.mv3 ENGINE = MergeTree ORDER BY tuple() AS SELECT c0 FROM mv3; -- { serverError BAD_ARGUMENTS, UNKNOWN_TABLE } +DROP TEMPORARY TABLE mv3; +DROP TABLE v; +RENAME TABLE mv TO mv2; +ALTER TABLE mv2 MODIFY QUERY SELECT c0 FROM mv2; -- { serverError BAD_ARGUMENTS } +CREATE TEMPORARY TABLE mv2 (c0 Int32); +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv2 MODIFY QUERY SELECT c0 FROM mv2; -- { serverError BAD_ARGUMENTS } +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv2 MODIFY QUERY SELECT c0 FROM src WHERE c0 IN mv2; -- { serverError BAD_ARGUMENTS } +DROP TEMPORARY TABLE mv2; +SELECT dependencies_table FROM system.tables WHERE database = currentDatabase() AND name = 'src'; + +-- A refreshable view is not triggered by inserts, so it may read itself. +CREATE TABLE dst (c0 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW rmv REFRESH EVERY 1 YEAR APPEND TO dst EMPTY AS SELECT c0 FROM src; +ALTER TABLE rmv MODIFY QUERY SELECT c0 FROM src WHERE c0 NOT IN (SELECT c0 FROM rmv); + +-- A string literal on the right side of IN in a column default is not a loading dependency either. +CREATE TABLE t1 (c0 String, c1 UInt8 DEFAULT c0 IN ('t2')) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (c0 String, c1 UInt8 DEFAULT c0 IN ('t1')) ENGINE = MergeTree ORDER BY tuple(); + +DROP TABLE t2; +DROP TABLE t1; +DROP TABLE rmv; +DROP TABLE dst; +DROP TABLE mv2; +DROP TABLE src; From 474f85828e8e06ff81d4eacfbcfb8dc684bc5fe2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 09:16:51 +0000 Subject: [PATCH 54/81] Backport #126204 to 26.8: Build the empty JOIN side of the constant-false short-circuit from its child's header --- .../shortCircuitConstantFalseJoin.cpp | 2 +- ...arallel_replicas_join_algorithms.reference | 5 +++ ...join_parallel_replicas_join_algorithms.sql | 33 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.reference create mode 100644 tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.sql diff --git a/src/Processors/QueryPlan/Optimizations/shortCircuitConstantFalseJoin.cpp b/src/Processors/QueryPlan/Optimizations/shortCircuitConstantFalseJoin.cpp index 4a4df418b4e82..9f5427104db53 100644 --- a/src/Processors/QueryPlan/Optimizations/shortCircuitConstantFalseJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/shortCircuitConstantFalseJoin.cpp @@ -80,7 +80,7 @@ size_t tryShortCircuitConstantFalseJoin(QueryPlan::Node * parent_node, QueryPlan return 0; auto & empty_node = nodes.emplace_back(); - empty_node.step = std::make_unique(join->getInputHeaders()[side]); + empty_node.step = std::make_unique(side_node->step->getOutputHeader()); parent_node->children[side] = &empty_node; return 1; }; diff --git a/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.reference b/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.reference new file mode 100644 index 0000000000000..05adba863b067 --- /dev/null +++ b/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.reference @@ -0,0 +1,5 @@ +hash RIGHT 2 +parallel_hash RIGHT spill 2 +grace_hash RIGHT 2 +join_use_nulls \N a +join_use_nulls \N c diff --git a/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.sql b/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.sql new file mode 100644 index 0000000000000..d4ea421606364 --- /dev/null +++ b/tests/queries/0_stateless/05401_short_circuit_false_join_parallel_replicas_join_algorithms.sql @@ -0,0 +1,33 @@ +-- Random settings limits: parallel_replicas_local_plan=(1, 1); query_plan_optimize_join_order_randomize=(0, 0) +-- A constant-false ON conjunct replaces the non-preserved side of a RIGHT JOIN with an empty source. +-- Plan-based parallel replicas optimize the local plan again after join runtime filters were added, +-- and every join algorithm must still get a usable header from that empty source. + +DROP TABLE IF EXISTS t_left; +DROP TABLE IF EXISTS t_right; + +CREATE TABLE t_left (b String) ENGINE = MergeTree ORDER BY b; +CREATE TABLE t_right (b String) ENGINE = MergeTree ORDER BY b; + +INSERT INTO t_left VALUES ('a'), ('b'); +INSERT INTO t_right VALUES ('a'), ('c'); + +SET enable_analyzer = 1; +SET automatic_parallel_replicas_mode = 0; +SET enable_parallel_replicas = 1; +SET parallel_replicas_for_non_replicated_merge_tree = 1; +SET max_parallel_replicas = 3; +SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +SET parallel_replicas_plan_based = 1; +SET enable_join_runtime_filters = 1; +SET query_plan_short_circuit_constant_false_join = 1; +SET max_bytes_before_external_join = 1000000000; + +SELECT 'hash RIGHT', count() FROM t_left AS l RIGHT JOIN t_right AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)) SETTINGS join_algorithm = 'hash'; +-- The in-memory join switches to grace_hash at the first block. +SELECT 'parallel_hash RIGHT spill', count() FROM t_left AS l RIGHT JOIN t_right AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)) SETTINGS join_algorithm = 'parallel_hash', max_bytes_before_external_join = 1; +SELECT 'grace_hash RIGHT', count() FROM t_left AS l RIGHT JOIN t_right AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)) SETTINGS join_algorithm = 'grace_hash'; +SELECT 'join_use_nulls', l.b, r.b FROM t_left AS l RIGHT JOIN t_right AS r ON l.b = r.b AND CAST(NULL AS Nullable(UInt8)) ORDER BY r.b SETTINGS join_algorithm = 'hash', join_use_nulls = 1; + +DROP TABLE t_left; +DROP TABLE t_right; From 3a9a506c6ab848099531f8a2e24dce6bea6ab335 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 20 Oct 2026 09:36:03 +0000 Subject: [PATCH 55/81] Adapt the Bool-literal selectivity hunk to 26.8 `set_for_ranges` comes from the set-range-sharing change on master, which is not in 26.8. Carry the IN set in a function-scope `SetPtr` so the expression path still sees the set's type. https://github.com/ClickHouse/ClickHouse/pull/126579 Co-Authored-By: Claude Opus 5.5 --- .../Statistics/ConditionSelectivityEstimator.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp index 0442d50574c4d..dac2fdcbdfaea 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp @@ -274,6 +274,9 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr Field const_value; DataTypePtr const_type; String column_name; + /// The IN set, when the atom is an IN whose set was turned into `const_value` below. + /// `const_type` stays null on that path, so this carries the element type instead. + SetPtr set_for_equal_factor; DataTypePtr column_type; if (node.isFunction()) @@ -384,6 +387,7 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr tuple[i] = (*columns[0])[i]; const_value = std::move(tuple); + set_for_equal_factor = prepared_set; column_name = func.getArgumentAt(0).getColumnName(); } else if (func.getArgumentAt(1).tryGetConstant(const_value, const_type)) @@ -427,8 +431,9 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr /// Skip range analysis to avoid bad cast when merging ranges of incompatible Field types. /// Pre-set selectivity based on the function type so prewhere ordering is still reasonable. /// Compared with a `Bool` literal or set, the expression is assumed to be boolean. - const Float64 equal_factor = set_for_ranges - ? getEqualFactor(set_for_ranges->getElementsTypes().front(), set_for_ranges->getTotalRowCount()) + const Float64 equal_factor = set_for_equal_factor + ? getEqualFactor( + set_for_equal_factor->getElementsTypes().front(), set_for_equal_factor->getTotalRowCount()) : getEqualFactor(const_type); if (func_name == "equals" || func_name == "in") out.selectivity.true_sel = equal_factor; From 36a25a2b3df5e75ea5cee70d0ca3de06232b6a1d Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 20 Oct 2026 09:41:58 +0000 Subject: [PATCH 56/81] Fix the 26.8 build of the row-mask coalescing backport `ASTSelectQuery::limit_after_all` does not exist in 26.8, and `splitAndModifyMutationCommands` takes `suitable_for_ttl_optimization` here. 26.8 ignores inline setting history in `DECLARE`, so the new settings are recorded in the 26.8 block of `SettingsChangesHistory.cpp`. https://github.com/ClickHouse/ClickHouse/pull/124274 Co-Authored-By: Claude Opus 5.5 --- src/Core/SettingsChangesHistory.cpp | 5 +++++ src/Storages/MergeTree/MergeTreeSettings.cpp | 15 +++++---------- src/Storages/MergeTree/MutateTask.cpp | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index cd304b3354214..188db3f5713b8 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -1454,6 +1454,11 @@ const VersionToSettingsChangesMap & getMergeTreeSettingsChangesHistory() addSettingsChanges(merge_tree_settings_changes_history, "26.8", { + {"enable_row_mask_update_coalescing", false, true, "Enable row-mask update coalescing by default"}, + {"max_row_mask_update_coalescing_keys", 256, 256, "New setting"}, + {"max_row_mask_update_coalescing_key_bytes", 64 * 1024, 64 * 1024, "New setting"}, + {"max_row_mask_update_coalescing_commands", 4096, 4096, "New setting"}, + {"max_row_mask_update_coalescing_ast_bytes", 1024 * 1024, 1024 * 1024, "New setting"}, {"shared_merge_tree_use_blobs_list_for_parts", false, false, "New setting which stores a SharedMergeTree part's per-file blob map in one consolidated Keeper node instead of one node per file"}, {"shared_merge_tree_blobs_list_inline_file_max_bytes", 0, 0, "New setting which stores small files of a blob-list part inline in the consolidated blobs.list instead of separate blobs"}, {"merge_use_batch_sorting_queue", false, false, "New setting to use the batch sorting queue for ordinary `MergeTree` merges."}, diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 5502513e81deb..832733cc8d86b 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -553,33 +553,28 @@ Coalesce adjacent singleton `UPDATE _row_exists = 0` commands while mutating a Wide part with Full storage. The persisted mutation commands and their versions are unchanged. Set to false to execute each command separately. -)", 0, \ - {"26.10", false, true, "Enable row-mask update coalescing by default"}) \ +)", 0) \ DECLARE(UInt64, max_row_mask_update_coalescing_keys, 256, R"( Maximum number of keys in one coalesced row-mask update. The effective limit also respects `max_rows_in_set` in the background query settings. Values below 2 disable coalescing. -)", 0, \ - {"26.10", 256, 256, "New setting"}) \ +)", 0) \ DECLARE(UInt64, max_row_mask_update_coalescing_key_bytes, 64 * 1024, R"( Maximum total number of key-literal bytes in one coalesced row-mask update. This bounds the size of the generated `IN` predicate. Set to 0 to disable coalescing. -)", 0, \ - {"26.10", 64 * 1024, 64 * 1024, "New setting"}) \ +)", 0) \ DECLARE(UInt64, max_row_mask_update_coalescing_commands, 4096, R"( Maximum number of part-local mutation commands eligible for row-mask update coalescing. This bounds the extra command-list copy made by the optimization; the value is a resource guard, not a semantic limit. Values below 2 disable coalescing. -)", 0, \ - {"26.10", 4096, 4096, "New setting"}) \ +)", 0) \ DECLARE(UInt64, max_row_mask_update_coalescing_ast_bytes, 1024 * 1024, R"( Maximum sum of the original commands' `ast_text` bytes eligible for row-mask update coalescing. The command-count limit alone cannot bound this copy when commands contain large literals. Set to 0 to disable coalescing. -)", 0, \ - {"26.10", 1024 * 1024, 1024 * 1024, "New setting"}) \ +)", 0) \ DECLARE(UInt64, max_number_of_merges_with_ttl_in_pool, 2, R"(When there is more than specified number of merges with TTL entries in pool, do not assign new merge with TTL. This is to leave free threads for regular merges and diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 0ada3db421781..cead60e122d59 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -200,7 +200,7 @@ static const ASTLiteral * getSingletonUInt128ArrayJoin(const ASTPtr & rhs) || select->group_by_with_totals || select->group_by_with_rollup || select->group_by_with_cube || select->group_by_with_constant_keys || select->group_by_with_grouping_sets || select->order_by_all - || select->limit_with_ties || select->limit_by_all || select->limit_after_all) + || select->limit_with_ties || select->limit_by_all) return nullptr; const auto * projections = select->select()->as(); @@ -4318,6 +4318,7 @@ bool MutateTask::prepare() *commands_for_execution, ctx->for_interpreter, unused_file_renames, + suitable_for_ttl_optimization, ctx->log); } From b31954f6236fa42e0686638cd5f1d1ee13365656 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:22:32 +0000 Subject: [PATCH 57/81] Remove the dictionary-definition checks from 05212 on 26.3 The show_create and system.query_log checks test that the url of an HTTP dictionary source is masked in the dictionary definition. On master that comes from #123364, which is not in 26.3, so both checks fail here. The runtime paths changed by #120407 stay covered. Co-Authored-By: Claude Opus 5.5 --- ...5212_http_uri_credentials_masked.reference | 1 - .../05212_http_uri_credentials_masked.sh | 22 ++----------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference index 229ae0059879a..b5d93d4ad7fa0 100644 --- a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference @@ -2,7 +2,6 @@ url_function: OK masked url_status_failure: OK masked insert_presigned: OK masked dictionary_reload: OK masked -show_create: OK masked schema_inference_cache: OK masked schema_inference_error: OK masked basic_header_passthrough: OK diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh index 92cfea296769d..a14ca8a6aeef2 100755 --- a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh @@ -5,9 +5,8 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -# A credential embedded in an HTTP URI must never appear in an error message, in SHOW CREATE output, -# or in system.query_log. Only the masked form scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for -# a presigned URL) may be shown. +# A credential embedded in an HTTP URI must never appear in an error message. Only the masked form +# scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for a presigned URL) may be shown. PW="pwleakprobe9f2a" URI="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping" @@ -58,23 +57,6 @@ ${CLICKHOUSE_CLIENT} --query "CREATE DICTIONARY dict_uri_leak (id UInt64, val St ${CLICKHOUSE_CLIENT} --query "SYSTEM RELOAD DICTIONARY dict_uri_leak" 2>&1 \ | grep -F 'in file/uri' | assert_shape "dictionary_reload" "$PW" "$USERINFO_SHAPE" -# 2b. SHOW CREATE must mask the password. -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DICTIONARY dict_uri_leak" 2>&1 | assert_shape "show_create" "$PW" "$USERINFO_SHAPE" - -# 3. system.query_log must store neither the query text nor the exception with the cleartext password. -# Print the offending rows rather than a count: on success this selects nothing (the reference has -# no line here), and any row printed on failure is the culprit - its query and exception show what -# leaked. The needle is split so that this checking query does not itself contain the contiguous -# secret. -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" -${CLICKHOUSE_CLIENT} --query " - SELECT type, query, exception - FROM system.query_log - WHERE event_date >= yesterday() - AND current_database = currentDatabase() - AND (query LIKE '%' || 'pwleakprobe' || '9f2a%' OR exception LIKE '%' || 'pwleakprobe' || '9f2a%') - FORMAT Vertical" - # 4. Schema inference stores the source URL in system.schema_inference_cache; the credential in its # userinfo must be masked there too. Inferring a schema from a credentialed /ping URL (which needs # no auth and whose "Ok." body is inferrable as one column) populates the cache. From cc51a7cc522b38bb7a20348510032e879e0e78b2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 17:06:28 +0000 Subject: [PATCH 58/81] Update autogenerated version to 26.8.32.14 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 67bf2f388cb48..f3356873a6869 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54544) +SET(VERSION_REVISION 54545) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 32) -SET(VERSION_GITHASH bb452427e21b3693aef652d597bb7fe4437b088a) -SET(VERSION_DESCRIBE v26.8.32.1-lts) -SET(VERSION_STRING 26.8.32.1) +SET(VERSION_PATCH 33) +SET(VERSION_GITHASH dc83db6f660bb74548f3309bbbf3bf4267988b66) +SET(VERSION_DESCRIBE v26.8.33.1-lts) +SET(VERSION_STRING 26.8.33.1) # end of autochange From 86429ed027564e138330b2ba3982e6312498c5fb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 19:30:54 +0000 Subject: [PATCH 59/81] Backport #126553 to 26.8: Fix use-after-free and reading of long values in complex key `ssd_cache` dictionaries --- src/Dictionaries/SSDCacheDictionaryStorage.h | 20 +++++++--- ...che_complex_key_reuse_partitions.reference | 4 ++ ..._ssd_cache_complex_key_reuse_partitions.sh | 37 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference create mode 100755 tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index f9ade57b4e312..3edd9815022c8 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -579,14 +579,16 @@ class SSDCacheFileBuffer : private boost::noncopyable return true; } - bool readKeys(size_t block_start, size_t blocks_length, PaddedPODArray & out) const + /// For complex keys, the keys written into `out` are `std::string_view`s pointing into `read_buffer_memory`, + /// so the caller must keep `read_buffer_memory` alive for as long as it uses `out`. + bool readKeys(size_t block_start, size_t blocks_length, Memory<> & read_buffer_memory, PaddedPODArray & out) const { if (block_start + blocks_length > current_blocks_size) return false; size_t buffer_size_in_bytes = blocks_length * block_size; - Memory<> read_buffer_memory(block_size * blocks_length, block_size); + read_buffer_memory = Memory<>(block_size * blocks_length, block_size); iocb request{}; iocb * request_ptr = &request; @@ -622,6 +624,9 @@ class SSDCacheFileBuffer : private boost::noncopyable throw ErrnoException(ErrorCodes::CANNOT_IO_GETEVENTS, "io_getevents: Failed to get an event for asynchronous IO"); } + // Unpoison the memory returned from an uninstrumented system function. + __msan_unpoison(&event, sizeof(event)); + auto read_bytes = eventResult(event); if (read_bytes != static_cast(buffer_size_in_bytes)) @@ -634,6 +639,9 @@ class SSDCacheFileBuffer : private boost::noncopyable ProfileEvents::increment(ProfileEvents::AIORead); ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes); + // Unpoison the memory returned from an uninstrumented system function. + __msan_unpoison(read_buffer_memory.data(), buffer_size_in_bytes); + SSDCacheBlock block(block_size); for (size_t i = 0; i < blocks_length; ++i) @@ -1150,8 +1158,8 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage for (auto & key_in_block : keys_in_block) { char * key_data = block_data + key_in_block.offset_in_block; - /// This is larger size than necessary but we don't know the exact size of serialized columns - size_t size = configuration.read_buffer_blocks_size; + /// The serialized columns are within the block, but we don't know their exact size. + size_t size = configuration.block_size - key_in_block.offset_in_block; ReadBufferFromString in({key_data, size}); deserializeAndInsertIntoColumns(result.fetched_columns, fetch_request, in); @@ -1304,8 +1312,10 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage if (started_reusing_old_partitions) { /// If we start reusing old partitions we need to remove old keys on disk from index before writing buffer + /// For complex keys, `old_keys` point into `old_keys_buffer`, so it must outlive them. + Memory<> old_keys_buffer; PaddedPODArray old_keys; - file_buffer.readKeys(block_index_in_file_before_write, configuration.write_buffer_blocks_size, old_keys); + file_buffer.readKeys(block_index_in_file_before_write, configuration.write_buffer_blocks_size, old_keys_buffer, old_keys); size_t file_read_end_block_index = block_index_in_file_before_write + configuration.write_buffer_blocks_size; diff --git a/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference new file mode 100644 index 0000000000000..388d823ad57f6 --- /dev/null +++ b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference @@ -0,0 +1,4 @@ +2000 2000 +2000 2000 +2000 1000 1000 +3000 diff --git a/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh new file mode 100755 index 0000000000000..2c4a7e813f02d --- /dev/null +++ b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `complex_key_ssd_cache` dictionary with tiny partitions, so that inserting keys wraps around the partitions +# and the storage reads the old keys back from the file to remove them from the index before overwriting them. +# The keys read back from the file must remain valid while they are used for the lookups in the index. +# The values read back from the file must not be truncated. + +$CLICKHOUSE_CLIENT --query=" + DROP DICTIONARY IF EXISTS ssd_cache_complex_key_reuse_dict; + DROP TABLE IF EXISTS ssd_cache_complex_key_reuse_source; + + CREATE TABLE ssd_cache_complex_key_reuse_source (k1 String, k2 UInt64, v String) ENGINE = Memory; + INSERT INTO ssd_cache_complex_key_reuse_source SELECT toString(number), number, 'value_' || toString(number) FROM numbers(3000); + + CREATE DICTIONARY ssd_cache_complex_key_reuse_dict + ( + k1 String, + k2 UInt64, + v String DEFAULT '' + ) + PRIMARY KEY k1, k2 + SOURCE(CLICKHOUSE(TABLE 'ssd_cache_complex_key_reuse_source' DB '${CLICKHOUSE_DATABASE}')) + LIFETIME(MIN 1000 MAX 1000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 4096 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 2 PATH '$USER_FILES_PATH/${CLICKHOUSE_DATABASE}_ssd_cache_complex_key_reuse')); + + SELECT count(), countIf(v = 'value_' || toString(number)) FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000)); + SELECT count(), countIf(v = 'value_' || toString(number)) FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000)); + SELECT count(), countIf(v = 'value_' || toString(number)), countIf(v = '') FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000, 2000)); + SELECT sum(dictHas('ssd_cache_complex_key_reuse_dict', (toString(number), number))) FROM numbers(4000); + + DROP DICTIONARY ssd_cache_complex_key_reuse_dict; + DROP TABLE ssd_cache_complex_key_reuse_source; +" From 8c3861f7b44884920680ca77e77fe9fa25f99c10 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 22:14:08 +0000 Subject: [PATCH 60/81] Backport #126548 to 26.8: Limit recursion depth when deserializing query plans --- src/Interpreters/ActionsDAG.cpp | 37 +- src/Interpreters/ActionsDAG.h | 8 +- src/Processors/QueryPlan/QueryPlan.h | 2 + src/Processors/QueryPlan/Serialization.cpp | 4 + .../QueryPlan/SetsSerialization.cpp | 15 +- ..._plan_deserialize_deep_recursion.reference | 3 + ...7_query_plan_deserialize_deep_recursion.sh | 320 ++++++++++++++++++ 7 files changed, 381 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference create mode 100755 tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 82c32b7cc7f56..e106fb126291f 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,7 @@ namespace ErrorCodes extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int BAD_ARGUMENTS; extern const int INCORRECT_DATA; + extern const int TOO_DEEP_RECURSION; } namespace @@ -4329,12 +4331,28 @@ static void serializeConstant(const IDataType & type, const IColumn & value, Wri type.getDefaultSerialization()->serializeBinary(data, 0, out, FormatSettings{}); } +/// The maximum nesting depth of lambda captures in a deserialized `ActionsDAG`. +/// The serialized data may come from a client, and the deserialization is recursive. +static constexpr size_t MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH = 1000; + +static void checkActionsDAGDeserializationDepth(size_t nesting_depth) +{ + checkStackSize(); + + if (nesting_depth > MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Too deep nesting of lambda captures in deserialized ActionsDAG (more than {} levels)", + MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH); +} + static ColumnConst::Ptr deserializeConstant( const IDataType & type, ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context, - size_t max_type_complexity) + size_t max_type_complexity, + size_t nesting_depth) { if (WhichDataType(type).isSet()) { @@ -4358,7 +4376,7 @@ static ColumnConst::Ptr deserializeConstant( { LambdaCapture capture; deserializeCapture(capture, in, max_type_complexity); - auto capture_dag = ActionsDAG::deserialize(in, registry, context, max_type_complexity); + auto capture_dag = ActionsDAG::deserialize(in, registry, context, max_type_complexity, nesting_depth + 1); UInt64 num_captured_columns = 0; readVarUInt(num_captured_columns, in); @@ -4367,7 +4385,7 @@ static ColumnConst::Ptr deserializeConstant( for (auto & captured_column : captured_columns) { captured_column.type = decodeDataType(in, max_type_complexity); - captured_column.column = deserializeConstant(*captured_column.type, in, registry, context, max_type_complexity); + captured_column.column = deserializeConstant(*captured_column.type, in, registry, context, max_type_complexity, nesting_depth + 1); /// `deserializeConstant` returns size-0 ColumnConsts to match the DAG node invariant, /// but a `ColumnFunction` requires its captured columns to share its `elements_size` /// (1 below) — `ColumnFunction::replicate` calls `replicate(offsets)` on each capture, @@ -4532,12 +4550,19 @@ void ActionsDAG::serialize(WriteBuffer & out, SerializedSetsRegistry & registry) writeVarUInt(node_to_id.at(output), out); } -ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context, size_t max_type_complexity) +ActionsDAG ActionsDAG::deserialize( + ReadBuffer & in, + DeserializedSetsRegistry & registry, + const ContextPtr & context, + size_t max_type_complexity, + size_t nesting_depth) { /// max_type_complexity is the type-complexity guard resolved once by the caller: the effective setting for /// client-reachable QueryPlan packets, or unlimited (0) for trusted internal metadata (e.g. data-lake /// schema transforms deserialized with the global context). + checkActionsDAGDeserializationDepth(nesting_depth); + size_t nodes_size = 0; readVarUInt(nodes_size, in); @@ -4577,7 +4602,7 @@ ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & r if ((column_flags & 2) == 0) node.is_deterministic_constant = false; - node.column = deserializeConstant(*node.result_type, in, registry, context, max_type_complexity); + node.column = deserializeConstant(*node.result_type, in, registry, context, max_type_complexity, nesting_depth); } if (node.type == ActionType::INPUT) @@ -4619,7 +4644,7 @@ ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & r { LambdaCapture capture; deserializeCapture(capture, in, max_type_complexity); - auto capture_dag = ActionsDAG::deserialize(in, registry, context, max_type_complexity); + auto capture_dag = ActionsDAG::deserialize(in, registry, context, max_type_complexity, nesting_depth + 1); node.function_base = std::make_shared( std::make_shared( diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index f9f60ca8aa4d7..4a96af748b929 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -167,7 +167,13 @@ class ActionsDAG /// max_type_complexity guards binary type decoding (0 == unlimited). Callers pass the effective /// input_format_binary_max_type_complexity for client-reachable QueryPlan packets, or leave it at the /// default 0 for trusted internal metadata (e.g. data-lake schema transforms). - static ActionsDAG deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context, size_t max_type_complexity = 0); + /// `nesting_depth` is the depth of lambda captures, it is used internally to limit the recursion. + static ActionsDAG deserialize( + ReadBuffer & in, + DeserializedSetsRegistry & registry, + const ContextPtr & context, + size_t max_type_complexity = 0, + size_t nesting_depth = 0); static Node createAlias(const Node & child, std::string alias); diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index ed67ba1a6526b..6c3e69ead3b95 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -237,6 +237,8 @@ class QueryPlan /// See `serializeForDistributedTask`. bool sets_must_be_ready = false; SizeLimits sets_transfer_limits = {}; + /// The nesting depth of plans for sets from subqueries, used on deserialize to limit the recursion. + size_t subquery_nesting_depth = 0; }; void serialize(WriteBuffer & out, const SerializationFlags & flags) const; diff --git a/src/Processors/QueryPlan/Serialization.cpp b/src/Processors/QueryPlan/Serialization.cpp index fc2ae2932e912..bf08e08aa2836 100644 --- a/src/Processors/QueryPlan/Serialization.cpp +++ b/src/Processors/QueryPlan/Serialization.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -213,6 +214,9 @@ QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & cont QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & context, const SerializationFlags & flags, size_t max_type_complexity) { + /// It is called recursively for plans of sets from subqueries. + checkStackSize(); + QueryPlanStepRegistry & step_registry = QueryPlanStepRegistry::instance(); DeserializedSetsRegistry sets_registry; diff --git a/src/Processors/QueryPlan/SetsSerialization.cpp b/src/Processors/QueryPlan/SetsSerialization.cpp index c4bab839c4cd2..6f85cf2e02506 100644 --- a/src/Processors/QueryPlan/SetsSerialization.cpp +++ b/src/Processors/QueryPlan/SetsSerialization.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes extern const int INCORRECT_DATA; extern const int SET_SIZE_LIMIT_EXCEEDED; extern const int SUPPORT_IS_DISABLED; + extern const int TOO_DEEP_RECURSION; } namespace Setting @@ -43,6 +44,10 @@ namespace Setting extern const SettingsOverflowMode transfer_overflow_mode; } +/// The maximum nesting depth of plans for sets from subqueries in a deserialized query plan. +/// The serialized plan may come from a client, and the deserialization is recursive. +static constexpr size_t MAX_SUBQUERY_PLAN_NESTING_DEPTH = 1000; + enum class SetSerializationKind : UInt8 { StorageSet = 1, @@ -242,7 +247,15 @@ QueryPlanAndSets QueryPlan::deserializeSets( } else if (kind == UInt8(SetSerializationKind::SubqueryPlan)) { - auto plan_for_set = QueryPlan::deserialize(in, context, flags, max_type_complexity); + auto nested_flags = flags; + ++nested_flags.subquery_nesting_depth; + if (nested_flags.subquery_nesting_depth > MAX_SUBQUERY_PLAN_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Too deep nesting of subqueries for sets in deserialized query plan (more than {} levels)", + MAX_SUBQUERY_PLAN_NESTING_DEPTH); + + auto plan_for_set = QueryPlan::deserialize(in, context, nested_flags, max_type_complexity); res.sets_from_subquery.emplace_back(QueryPlanAndSets::SetFromSubquery{ {hash, std::move(columns)}, diff --git a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference new file mode 100644 index 0000000000000..c214d11a9befd --- /dev/null +++ b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference @@ -0,0 +1,3 @@ +shallow plan: no TOO_DEEP_RECURSION +TOO_DEEP_RECURSION +server alive 1 diff --git a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh new file mode 100755 index 0000000000000..2e7b1048f819d --- /dev/null +++ b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh @@ -0,0 +1,320 @@ +#!/usr/bin/env bash +# Regression test for unbounded recursion in `ActionsDAG::deserialize`. +# +# A `QueryPlan` packet (native-protocol packet type 13) can carry an `ActionsDAG` with a +# lambda-capture constant whose capture DAG is itself another lambda-capture constant, and so +# on. `ActionsDAG::deserialize` / `deserializeConstant` recurse once per nesting level, so a +# client that sends a few thousand nested captures used to exhaust the server stack and take the +# whole process down with SIGSEGV. The fix caps the nesting depth and adds `checkStackSize`, +# turning the attack into a `TOO_DEEP_RECURSION` exception while the server keeps serving. +# +# The packet is only reachable for a query sent with the `QueryPlan` processing stage while the +# server has `process_query_plan_packet` enabled (see tests/config/config.d/process_query_plan_packet.xml). +# The client here speaks the raw native protocol pinned to revision 54482 so the revision-gated +# fields of the handshake are fixed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 - <<'PYEOF' +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_USER = os.environ.get("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = os.environ.get("CLICKHOUSE_PASSWORD", "") +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") + +CLIENT_NAME = "query plan deep recursion poc" +REVISION = 54482 + +# Plan serialization version gates for header fields (see Core/ProtocolDefines.h). +WITH_EXECUTION_LIMITS = 11 # max_threads + concurrency_control +WITH_STEP_VERSIONS = 19 # per-step version + +# ActionsDAG::ActionType (Interpreters/ActionsDAG.h) +ACTION_COLUMN = 1 + +# BinaryTypeIndex (DataTypes/DataTypesBinaryEncoding.h) +TYPE_UINT8 = b"\x01" +# Function(0 args) -> UInt8 : 0x24 +TYPE_FUNCTION = b"\x24\x00" + TYPE_UINT8 + +DEPTH = 5000 +SHALLOW_DEPTH = 10 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def varuint(x): + ba = bytearray() + writeVarUInt(x, ba) + return bytes(ba) + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def strbin(s): + ba = bytearray() + writeStringBinary(s, ba) + return bytes(ba) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary(CLICKHOUSE_USER, ba) + writeStringBinary(CLICKHOUSE_PASSWORD, ba) + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + readStringBinary(s) # proto_send chunked + readStringBinary(s) # proto_recv chunked + for _ in range(readVarUInt(s)): # password complexity rules + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce + while True: # server settings + if readStringBinary(s) == "": + break + readVarUInt(s) + readStringBinary(s) + query_plan_version = readVarUInt(s) # query plan serialization version + readVarUInt(s) # cluster function protocol version + return query_plan_version + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key + writeStringBinary("notchunked", ba) # proto_send chunked + writeStringBinary("notchunked", ba) # proto_recv chunked + writeVarUInt(0, ba) # parallel replicas protocol version + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary(CLICKHOUSE_USER, ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client version patch + ba.append(0) # opentelemetry: no trace id + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number + writeVarUInt(0, ba) # script line number + ba.append(0) # jwt: none + + +def sendQuery(s, query, stage): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles + writeStringBinary("", ba) # interserver secret + writeVarUInt(stage, ba) # query processing stage + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator + s.sendall(ba) + + +def buildActionsDAG(depth): + """An ActionsDAG with `depth` nested lambda-capture constants. + + Each level is a single COLUMN node of a Function type; its constant is a lambda capture + whose capture DAG is the next level. The innermost level is a plain UInt8 constant, so a + correct (shallow) deserializer would terminate. + """ + # Terminal: one COLUMN node holding a UInt8 constant. + dag = bytearray() + dag += varuint(1) # nodes_size + dag += bytes([ACTION_COLUMN]) + dag += strbin("") # result_name + dag += TYPE_UINT8 # result_type + dag += varuint(0) # children + dag += bytes([1]) # column_flags: has column + dag += bytes([0]) # UInt8 constant value + dag += varuint(0) # inputs + dag += varuint(0) # outputs + + for _ in range(depth): + unit = bytearray() + unit += varuint(1) # nodes_size + unit += bytes([ACTION_COLUMN]) + unit += strbin("") # result_name + unit += TYPE_FUNCTION # result_type -> isFunction() branch in deserializeConstant + unit += varuint(0) # children + unit += bytes([1]) # column_flags: has column + # deserializeCapture + unit += strbin("") # capture.return_name + unit += TYPE_UINT8 # capture.return_type + unit += varuint(0) # num captured names + unit += varuint(0) # num captured types + unit += varuint(0) # num lambda arguments + # recursive ActionsDAG::deserialize for the capture DAG + unit += dag + unit += varuint(0) # num_captured_columns + unit += varuint(0) # inputs + unit += varuint(0) # outputs + dag = unit + + return bytes(dag) + + +def buildQueryPlanPacket(plan_version, depth): + body = bytearray() + writeVarUInt(plan_version, body) + if plan_version >= WITH_EXECUTION_LIMITS: + writeVarUInt(0, body) # max_threads + body.append(0) # concurrency_control + # Node tree, single root Expression step with no children. + writeVarUInt(0, body) # num_children + writeStringBinary("Expression", body) + if plan_version >= WITH_STEP_VERSIONS: + writeVarUInt(0, body) # step_version + writeStringBinary("", body) # step_description + writeVarUInt(0, body) # output header: 0 columns + writeVarUInt(0, body) # settings: 0 changed + # ExpressionStep::deserialize -> ActionsDAG::deserialize + body += buildActionsDAG(depth) + + packet = bytearray() + writeVarUInt(13, packet) # Client::QueryPlan + packet += body + return bytes(packet) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readStrict(s) # has_nested + return code, text + + +def sendPlan(depth): + """Sends a `QueryPlan` packet with `depth` nested lambda captures and returns the exception code and text, or None if the query is not rejected.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(60) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + plan_version = receiveHello(s) + sendAddendum(s) + # Stage 7 == QueryProcessingStage::QueryPlan: the server then expects a QueryPlan packet. + sendQuery(s, "SELECT 1", 7) + s.sendall(buildQueryPlanPacket(plan_version, depth)) + + # Only the first packet matters: either the exception or the start of the query result. + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s) + return None, "packet {}".format(packet_type) + + +def main(): + # A shallow plan must not hit the recursion limit: it distinguishes the depth limit from rejecting any plan. + code, text = sendPlan(SHALLOW_DEPTH) + if code == 306: + print("UNEXPECTED TOO_DEEP_RECURSION for a shallow plan: {}".format(text)) + else: + print("shallow plan: no TOO_DEEP_RECURSION") + + code, text = sendPlan(DEPTH) + # 306 == TOO_DEEP_RECURSION. In sanitizer builds, `checkStackSize` may throw it before the depth limit is reached. + if code == 306: + print("TOO_DEEP_RECURSION") + else: + print("UNEXPECTED EXCEPTION {}: {}".format(code, text)) + + +main() +PYEOF + +# The server must still be alive and serving after rejecting the malicious packet. +${CLICKHOUSE_CLIENT} --query "SELECT 'server alive', 1" From 9f3e5acb00e07f94943a9dd1a5bc22beab7c9672 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 23:44:39 +0000 Subject: [PATCH 61/81] Backport #113599 to 26.8: Make the columns-description-cache eviction tests robust to asynchronous part removal --- .../03673_columns_description_cache.reference | 26 +----- .../03673_columns_description_cache.sh | 82 +++++++++++++++++++ .../03673_columns_description_cache.sql | 31 ------- ...scriptions_cache_nested_eviction.reference | 12 +-- ...umns_descriptions_cache_nested_eviction.sh | 59 +++++++++++++ ...mns_descriptions_cache_nested_eviction.sql | 19 ----- 6 files changed, 147 insertions(+), 82 deletions(-) create mode 100755 tests/queries/0_stateless/03673_columns_description_cache.sh delete mode 100644 tests/queries/0_stateless/03673_columns_description_cache.sql create mode 100755 tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sh delete mode 100644 tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sql diff --git a/tests/queries/0_stateless/03673_columns_description_cache.reference b/tests/queries/0_stateless/03673_columns_description_cache.reference index 77f94a2349d47..9417fcd54077a 100644 --- a/tests/queries/0_stateless/03673_columns_description_cache.reference +++ b/tests/queries/0_stateless/03673_columns_description_cache.reference @@ -1,32 +1,14 @@ +-- the cache is only for MergeTree 0 --- { echoOn } -create table t_mt (key Int) engine=MergeTree() order by (); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; +-- MergeTree: an entry per distinct column list of the alive parts 0 -insert into t_mt values (1); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; 1 -insert into t_mt values (2); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; 1 -alter table t_mt add column value String settings mutations_sync=2; -insert into t_mt values (10, '10'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; 2 -insert into t_mt values (20, '20'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; 2 --- now let's try to remove ColumnsDescription with old structure -alter table t_mt detach part 'all_1_1_0'; -alter table t_mt detach part 'all_2_2_0'; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; +-- the entry of the old structure must be evicted once both its parts are gone 1 --- reattach -detach table t_mt; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -attach table t_mt; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; +-- and after a detach/attach round trip of the table 1 -- system.metrics -select value > 0 from system.metrics where metric = 'ColumnsDescriptionsCacheSize'; 1 diff --git a/tests/queries/0_stateless/03673_columns_description_cache.sh b/tests/queries/0_stateless/03673_columns_description_cache.sh new file mode 100755 index 0000000000000..48e06955f517b --- /dev/null +++ b/tests/queries/0_stateless/03673_columns_description_cache.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Tags: no-shared-merge-tree, memory-engine + +# The per-table cache of shared part column descriptions: entries appear as parts with a new +# schema are added and disappear once the last part holding a column list is gone. +# +# DETACH PART is asynchronous in two ways (the detached part is replaced by a new empty covering +# part and the replaced part object is removed from memory by background cleanup, and a +# detach+attach of the table can even reload the not-yet-removed directory as a covered part), so +# the assertions after it poll until the expected value with a short parts lifetime. The +# assertions established synchronously by the preceding statement stay exact reads. +# Merges are disabled because the parts below are detached by name. +# Tagged no-shared-merge-tree: the eviction under test is driven by the lifecycle of local part +# objects, which SharedMergeTree manages through Keeper on its own schedule, so the polls do not +# converge there within the test time budget. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function cleanup() +{ + # DROP TABLE IF EXISTS only sees attached tables, so a table left detached by a run that died + # between DETACH TABLE and ATTACH TABLE survives it and makes the next CREATE TABLE fail with + # TABLE_ALREADY_EXISTS (detached). ATTACH TABLE IF NOT EXISTS throws without metadata at all. + local sql + sql=$(${CLICKHOUSE_CLIENT} --query "SELECT 'ATTACH TABLE ' || table || ';' FROM system.detached_tables WHERE database = currentDatabase() AND table = 't_mt'") + ${CLICKHOUSE_CLIENT} --query "$sql DROP TABLE IF EXISTS t_mem; DROP TABLE IF EXISTS t_mt;" +} + +function cache_size() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT columns_descriptions_cache_size FROM system.tables WHERE database = currentDatabase() AND table = '$1'" +} + +# Prints the cache size of table $1 once it reaches $2 (or the last observed value on timeout, +# so a failure shows the actual state). +function wait_for_cache_size() +{ + local res + for _ in {1..200} + do + res=$(cache_size "$1") + [ "$res" = "$2" ] && break + sleep 0.3 + done + echo "$res" +} + +cleanup +trap cleanup EXIT + +echo "-- the cache is only for MergeTree" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE t_mem (key Int) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_mem VALUES (1)" +cache_size t_mem + +echo "-- MergeTree: an entry per distinct column list of the alive parts" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE t_mt (key Int) ENGINE = MergeTree ORDER BY () SETTINGS old_parts_lifetime = 1, max_bytes_to_merge_at_max_space_in_pool = 1" +cache_size t_mt +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_mt VALUES (1)" +cache_size t_mt +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_mt VALUES (2)" +cache_size t_mt +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_mt ADD COLUMN value String SETTINGS mutations_sync = 2" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_mt VALUES (10, '10')" +cache_size t_mt +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_mt VALUES (20, '20')" +cache_size t_mt + +echo "-- the entry of the old structure must be evicted once both its parts are gone" +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_mt DETACH PART 'all_1_1_0'" +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_mt DETACH PART 'all_2_2_0'" +wait_for_cache_size t_mt 1 + +echo "-- and after a detach/attach round trip of the table" +${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_mt" +${CLICKHOUSE_CLIENT} --query "ATTACH TABLE t_mt" +wait_for_cache_size t_mt 1 + +echo "-- system.metrics" +${CLICKHOUSE_CLIENT} --query "SELECT value > 0 FROM system.metrics WHERE metric = 'ColumnsDescriptionsCacheSize'" diff --git a/tests/queries/0_stateless/03673_columns_description_cache.sql b/tests/queries/0_stateless/03673_columns_description_cache.sql deleted file mode 100644 index 925d90bb57c18..0000000000000 --- a/tests/queries/0_stateless/03673_columns_description_cache.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Cache is only for MergeTree -drop table if exists t_mem; -create table t_mem (key Int) engine=Memory(); -insert into t_mem values (1); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mem'; - --- MergeTree -drop table if exists t_mt; --- { echoOn } -create table t_mt (key Int) engine=MergeTree() order by (); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -insert into t_mt values (1); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -insert into t_mt values (2); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -alter table t_mt add column value String settings mutations_sync=2; -insert into t_mt values (10, '10'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -insert into t_mt values (20, '20'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; --- now let's try to remove ColumnsDescription with old structure -alter table t_mt detach part 'all_1_1_0'; -alter table t_mt detach part 'all_2_2_0'; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; --- reattach -detach table t_mt; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; -attach table t_mt; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_mt'; --- system.metrics -select value > 0 from system.metrics where metric = 'ColumnsDescriptionsCacheSize'; diff --git a/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.reference b/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.reference index 8149f64ababe2..2d1a9340f12ec 100644 --- a/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.reference +++ b/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.reference @@ -1,14 +1,6 @@ --- { echoOn } -create table t_nested_leak (key Int, `n.a` Array(Int32), `n.b` Array(String)) engine=MergeTree() order by key; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; +-- an entry per distinct column list of the alive parts 0 -insert into t_nested_leak values (1, [10], ['hello']); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; 1 -alter table t_nested_leak add column value String settings mutations_sync=2; -insert into t_nested_leak values (10, [30], ['!'], '10'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; 2 -alter table t_nested_leak detach part 'all_1_1_0'; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; +-- the entry of the detached part must be evicted once its object is gone 1 diff --git a/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sh b/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sh new file mode 100755 index 0000000000000..113e16700cd72 --- /dev/null +++ b/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tags: no-shared-merge-tree + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/102926: +# `MergeTreeData::releaseSharedPartColumns` must evict cache entries for schemas with `Nested` +# columns, where `Nested::collect` produces a distinct `with_collected_nested` description. +# +# DETACH PART is asynchronous (the detached part is replaced by a new empty covering part and the +# replaced part object is removed from memory by background cleanup), so the assertion after it +# polls until the expected value with a short parts lifetime. The assertions established +# synchronously by the preceding statement stay exact reads. +# Merges are disabled because the part below is detached by name. +# Tagged no-shared-merge-tree: the eviction under test is driven by the lifecycle of local part +# objects, which SharedMergeTree manages through Keeper on its own schedule, so the polls do not +# converge there within the test time budget. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t_nested_leak" +} + +function cache_size() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT columns_descriptions_cache_size FROM system.tables WHERE database = currentDatabase() AND table = '$1'" +} + +# Prints the cache size of table $1 once it reaches $2 (or the last observed value on timeout, +# so a failure shows the actual state). +function wait_for_cache_size() +{ + local res + for _ in {1..200} + do + res=$(cache_size "$1") + [ "$res" = "$2" ] && break + sleep 0.3 + done + echo "$res" +} + +cleanup +trap cleanup EXIT + +echo "-- an entry per distinct column list of the alive parts" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE t_nested_leak (key Int, \`n.a\` Array(Int32), \`n.b\` Array(String)) ENGINE = MergeTree ORDER BY key SETTINGS old_parts_lifetime = 1, max_bytes_to_merge_at_max_space_in_pool = 1" +cache_size t_nested_leak +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_nested_leak VALUES (1, [10], ['hello'])" +cache_size t_nested_leak +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_nested_leak ADD COLUMN value String SETTINGS mutations_sync = 2" +${CLICKHOUSE_CLIENT} --query "INSERT INTO t_nested_leak VALUES (10, [30], ['!'], '10')" +cache_size t_nested_leak + +echo "-- the entry of the detached part must be evicted once its object is gone" +${CLICKHOUSE_CLIENT} --query "ALTER TABLE t_nested_leak DETACH PART 'all_1_1_0'" +wait_for_cache_size t_nested_leak 1 diff --git a/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sql b/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sql deleted file mode 100644 index 3972722cfcdd2..0000000000000 --- a/tests/queries/0_stateless/04257_102926_columns_descriptions_cache_nested_eviction.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Regression test for https://github.com/ClickHouse/ClickHouse/issues/102926 --- `decrefColumnsDescriptionForColumns` must evict cache entries for schemas with --- `Nested` columns where `Nested::collect` produces a distinct `with_collected_nested`. - -drop table if exists t_nested_leak; - --- { echoOn } -create table t_nested_leak (key Int, `n.a` Array(Int32), `n.b` Array(String)) engine=MergeTree() order by key; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; -insert into t_nested_leak values (1, [10], ['hello']); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; -alter table t_nested_leak add column value String settings mutations_sync=2; -insert into t_nested_leak values (10, [30], ['!'], '10'); -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; -alter table t_nested_leak detach part 'all_1_1_0'; -select columns_descriptions_cache_size from system.tables where database = currentDatabase() and table = 't_nested_leak'; --- { echoOff } - -drop table t_nested_leak; From 4c739d7e2100407833bda7cdf62f45bdf5bc1862 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 00:25:13 +0000 Subject: [PATCH 62/81] Backport #126942 to 26.8: Fix wrong `count()` for `Date32` equality to `1900-01-01` in the sparsity trivial-count rewrite --- src/Core/Settings.cpp | 2 +- src/Storages/MergeTree/SparsityFilter.cpp | 5 ++- ...sparsity_date32_default_not_zero.reference | 4 +++ ...05427_sparsity_date32_default_not_zero.sql | 33 +++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.reference create mode 100644 tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.sql diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index ac09d124a1734..3beb8564864b8 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -4997,7 +4997,7 @@ Patterns recognised: - `col > 0`, `col >= 1`, `col < 1`, `col <= 0` on unsigned integer columns. - Bare `col` / `NOT col` on `Int*`, `UInt*`, `Bool` columns (truthy test). -The equality patterns are not applied to `Float*`, `Enum*`, `Nullable`, `LowCardinality`, +The equality patterns are not applied to `Float*`, `Enum*`, `Date32`, `Nullable`, `LowCardinality`, or composite types (`Tuple`, `Array`, `Map`, ...) — for these the count is served from the regular scan path. diff --git a/src/Storages/MergeTree/SparsityFilter.cpp b/src/Storages/MergeTree/SparsityFilter.cpp index 73dc7c322df6a..2a640f253b09c 100644 --- a/src/Storages/MergeTree/SparsityFilter.cpp +++ b/src/Storages/MergeTree/SparsityFilter.cpp @@ -63,8 +63,8 @@ bool isStringLike(const DataTypePtr & type) /// `num_defaults` is produced by `IColumn::isDefaultAt`, which is bitwise: row is /// counted iff its underlying storage equals zero. For these types SQL equality to -/// `default(type)` matches the same row set. Excluded: Enum (type default is a named -/// value, not bitwise 0), Float (treats -0.0 and +0.0 separately, handled at the +/// `default(type)` matches the same row set. Excluded: Enum and Date32 (type default is not bitwise 0: +/// a named value, 1900-01-01), Float (treats -0.0 and +0.0 separately, handled at the /// caller), Nullable / LowCardinality / composite (Tuple, Array, Map, ...) where the /// "default" partition does not coincide with SQL equality to the type default. bool typeMatchesDefaultnessStat(const DataTypePtr & type) @@ -77,7 +77,6 @@ bool typeMatchesDefaultnessStat(const DataTypePtr & type) || w.isString() || w.isFixedString() || w.isDate() - || w.isDate32() || w.isDateTime() || w.isDateTime64() || w.isDecimal() diff --git a/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.reference b/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.reference new file mode 100644 index 0000000000000..c98b8a242d60d --- /dev/null +++ b/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.reference @@ -0,0 +1,4 @@ +eq 20 +eq_reversed 20 +ne 980 +dense_eq 1 diff --git a/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.sql b/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.sql new file mode 100644 index 0000000000000..6283826244737 --- /dev/null +++ b/tests/queries/0_stateless/05427_sparsity_date32_default_not_zero.sql @@ -0,0 +1,33 @@ +-- `Date32` default is 1900-01-01, but the per-part `num_defaults` counter counts bitwise-zero rows (1970-01-01), +-- so `count()` with an equality to the `Date32` default must not be served from that counter. +SET optimize_trivial_count_query = 1; + +DROP TABLE IF EXISTS t_date32_sparse; +DROP TABLE IF EXISTS t_date32_dense; + +CREATE TABLE t_date32_sparse (id UInt32, d Date32) +ENGINE = MergeTree ORDER BY id +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5, + compute_exact_num_defaults_for_sparse_columns = 1, + serialization_info_version = 'with_types', + min_bytes_for_wide_part = 0; + +INSERT INTO t_date32_sparse SELECT number, if(number % 50 = 7, toDate32('1900-01-01'), toDate32('1970-01-01')) FROM numbers(1000); + +SELECT 'eq', count() FROM t_date32_sparse WHERE d = toDate32('1900-01-01') SETTINGS optimize_trivial_count_with_sparsity_filter = 1; +SELECT 'eq_reversed', count() FROM t_date32_sparse WHERE toDate32('1900-01-01') = d SETTINGS optimize_trivial_count_with_sparsity_filter = 1; +SELECT 'ne', count() FROM t_date32_sparse WHERE d != toDate32('1900-01-01') SETTINGS optimize_trivial_count_with_sparsity_filter = 1; + +CREATE TABLE t_date32_dense (d Date32) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5, + compute_exact_num_defaults_for_sparse_columns = 1, + serialization_info_version = 'with_types', + min_bytes_for_wide_part = 0; + +INSERT INTO t_date32_dense VALUES ('1900-01-01'), ('2000-02-29'); + +SELECT 'dense_eq', count() FROM t_date32_dense WHERE d = toDate32('1900-01-01') SETTINGS optimize_trivial_count_with_sparsity_filter = 1; + +DROP TABLE t_date32_sparse; +DROP TABLE t_date32_dense; From bd9a52d2b7b7fb120c076bcbbd4f0a4b5c4e8c39 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 04:49:07 +0000 Subject: [PATCH 63/81] Backport #127015 to 26.8: Fix reading uninitialized memory in `polygonConvexHullCartesian` --- src/Functions/polygonConvexHull.cpp | 19 ++++++++++++++++++- ..._convex_hull_empty_exterior_ring.reference | 6 ++++++ ...olygon_convex_hull_empty_exterior_ring.sql | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference create mode 100644 tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index a903dc11b5077..681fd1a094d9a 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -2,6 +2,7 @@ #include +#include #include namespace DB @@ -15,6 +16,21 @@ namespace ErrorCodes namespace { +/// `boost::geometry::convex_hull` takes only the exterior rings of polygons into account, +/// but it checks for emptiness with `boost::geometry::is_empty`, which also looks at the interior rings. +/// For a polygon with an empty exterior ring and a non-empty interior ring it reads uninitialized points. +/// Returns true if there are no points for the convex hull to be built from. +template +bool hasNoPointsForConvexHull(const Geometry & geometry) +{ + if constexpr (std::is_same_v, boost::geometry::polygon_tag>) + return geometry.outer().empty(); + else if constexpr (std::is_same_v, boost::geometry::multi_polygon_tag>) + return std::ranges::all_of(geometry, [](const auto & polygon) { return polygon.outer().empty(); }); + else + return boost::geometry::is_empty(geometry); +} + template class FunctionPolygonConvexHull final : public IFunction { @@ -68,7 +84,8 @@ class FunctionPolygonConvexHull final : public IFunction for (size_t i = 0; i < input_rows_count; ++i) { Polygon convex_hull{}; - boost::geometry::convex_hull(geometries[i], convex_hull); + if (!hasNoPointsForConvexHull(geometries[i])) + boost::geometry::convex_hull(geometries[i], convex_hull); serializer.add(convex_hull); } } diff --git a/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference new file mode 100644 index 0000000000000..0304a56da9fbe --- /dev/null +++ b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference @@ -0,0 +1,6 @@ +[[]] +[[]] +[[]] +POLYGON((0 0,0 5,5 5,5 0,0 0)) +[[]] +[[(0,0),(0,5),(5,5),(0,0)]] diff --git a/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql new file mode 100644 index 0000000000000..0d0036254a2a6 --- /dev/null +++ b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql @@ -0,0 +1,6 @@ +-- A polygon with an empty exterior ring and a non-empty interior ring led to reading uninitialized memory. +SELECT polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.), (2., 0.)]]]::MultiPolygon); +SELECT polygonConvexHullCartesian([[], [(0., 0.), (1., 1.), (2., 0.)]]::Polygon); +SELECT polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.)]], [[], [(5., 5.), (6., 6.)]]]::MultiPolygon); +SELECT wkt(polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.)]], [[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (2., 3.)]]]::MultiPolygon)); +SELECT polygonConvexHullCartesian(arrayJoin([[[[], [(0., 0.), (1., 1.)]]], [[[(0., 0.), (0., 5.), (5., 5.)]]]])::MultiPolygon); From 8f61a7ee625b5430d24aabdace2daa33a7107cf5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 21:46:15 +0000 Subject: [PATCH 64/81] Backport #125701 to 26.8: Require READ and WRITE privileges for the FileLog table engine --- src/Storages/FileLog/StorageFileLog.cpp | 1 + ...filelog_engine_file_source_grant.reference | 6 +++ .../05394_filelog_engine_file_source_grant.sh | 40 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference create mode 100755 tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 926522eccbc03..197b1c758f5e3 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -906,6 +906,7 @@ void registerStorageFileLog(StorageFactory & factory) creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, + .source_access_type = AccessTypeObjects::Source::FILE, .has_builtin_setting_fn = FileLogSettings::hasBuiltin, }, Documentation{ diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference new file mode 100644 index 0000000000000..d686d2c8371c1 --- /dev/null +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference @@ -0,0 +1,6 @@ +-- no FILE grant: CREATE is refused, nothing is left behind +ACCESS_DENIED +0 +-- READ and WRITE on FILE: CREATE and SELECT work +owned_marker +secret_payload diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh new file mode 100755 index 0000000000000..e12f25a2608fa --- /dev/null +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `FileLog` table reads files under `user_files_path`, the same data the `file` function and the +# `File` engine guard with the `FILE` source grant. The engine therefore declares `FILE` as its source, +# so `TABLE ENGINE ON FileLog` is no longer handed to everyone and a user without the source grant is +# refused at `CREATE TABLE` time instead of reading the staged files. + +# A user and a file in `user_files` are global names, and the flaky check runs several copies of one +# test at once, so they carry the pid as well as the unique name. +NAME="${CLICKHOUSE_TEST_UNIQUE_NAME}_$$" +USER="low_${NAME}" +FILE="${NAME}.csv" +TABLE="${CLICKHOUSE_DATABASE}.fl_${NAME}" + +# Match the privilege sentence rather than the engine name: the client echoes the failing query, which +# itself contains `FileLog`. +denied() { grep -qF "necessary to have the grant TABLE ENGINE ON FileLog"; } + +printf 'owned_marker\nsecret_payload\n' > "${USER_FILES_PATH:?}/${FILE}" + +${CLICKHOUSE_CLIENT} -q "DROP USER IF EXISTS ${USER}" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER}" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE, SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${USER}" + +echo "-- no FILE grant: CREATE is refused, nothing is left behind" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'fl_${NAME}'" + +echo "-- READ and WRITE on FILE: CREATE and SELECT work" +${CLICKHOUSE_CLIENT} -q "GRANT READ, WRITE ON FILE TO ${USER}" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "SELECT marker FROM ${TABLE} ORDER BY marker SETTINGS stream_like_engine_allow_direct_select = 1" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP USER ${USER}" +rm -f "${USER_FILES_PATH:?}/${FILE}" From 37abf455e089f7ef0ae70adefdcf38ef506dd013 Mon Sep 17 00:00:00 2001 From: Oranje AI <293843428+oranjeai@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:26:08 +0100 Subject: [PATCH 65/81] Pass an absolute `FileLog` path in `05394_filelog_engine_file_source_grant` on 26.8 The test passed a bare file name to `FileLog`. Master resolves a relative `FileLog` path against `user_files_path` since https://github.com/ClickHouse/ClickHouse/pull/123397, which is not on 26.8. Here the name stays relative to the server working directory, so the `StorageFileLog` constructor threw `BAD_ARGUMENTS` "The absolute data path should be inside `user_files_path`" once the grant check passed. Use `${USER_FILES_PATH}/` like the other `FileLog` tests on 26.8. The privilege check runs before the engine is constructed, so the first half still verifies the `ACCESS_DENIED` without the `FILE` grant. Same change @fm4v merged on the 26.4 leg as 737ad1dbe626; only the branch number in the comment differs. --- .../0_stateless/05394_filelog_engine_file_source_grant.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh index e12f25a2608fa..a0028acc6fff4 100755 --- a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh @@ -13,6 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # test at once, so they carry the pid as well as the unique name. NAME="${CLICKHOUSE_TEST_UNIQUE_NAME}_$$" USER="low_${NAME}" +# On 26.8 `FileLog` does not resolve a relative path against `user_files_path`, so pass it absolute. FILE="${NAME}.csv" TABLE="${CLICKHOUSE_DATABASE}.fl_${NAME}" @@ -27,12 +28,12 @@ ${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER}" ${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE, SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${USER}" echo "-- no FILE grant: CREATE is refused, nothing is left behind" -${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${USER_FILES_PATH}/${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'fl_${NAME}'" echo "-- READ and WRITE on FILE: CREATE and SELECT work" ${CLICKHOUSE_CLIENT} -q "GRANT READ, WRITE ON FILE TO ${USER}" -${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${USER_FILES_PATH}/${FILE}', 'LineAsString')" ${CLICKHOUSE_CLIENT} --user "${USER}" -q "SELECT marker FROM ${TABLE} ORDER BY marker SETTINGS stream_like_engine_allow_direct_select = 1" ${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" From 8a78545d387a8ce7ac7f4f3b35990cd41d6cb451 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Oct 2026 11:06:13 +0000 Subject: [PATCH 66/81] Fix `05427_query_plan_deserialize_deep_recursion` for 26.8 plan serialization version On 26.8, `DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS` is 10 (it is 11 on master), and the server advertises plan version 10. The test did not write `max_threads` / `concurrency_control` for version 10, so the hand-crafted packet was misaligned and the server failed with an unrelated exception (`Method createColumn is not implemented for data type Function( -> UInt8)`) instead of `TOO_DEEP_RECURSION`. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=127048&sha=8c3861f7b44884920680ca77e77fe9fa25f99c10&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 PR: https://github.com/ClickHouse/ClickHouse/pull/127048 Co-Authored-By: Claude Opus 5.5 --- .../0_stateless/05427_query_plan_deserialize_deep_recursion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh index 2e7b1048f819d..5cb4e75d2298b 100755 --- a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh +++ b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh @@ -32,7 +32,7 @@ CLIENT_NAME = "query plan deep recursion poc" REVISION = 54482 # Plan serialization version gates for header fields (see Core/ProtocolDefines.h). -WITH_EXECUTION_LIMITS = 11 # max_threads + concurrency_control +WITH_EXECUTION_LIMITS = 10 # max_threads + concurrency_control WITH_STEP_VERSIONS = 19 # per-step version # ActionsDAG::ActionType (Interpreters/ActionsDAG.h) From b1beeb724d337e7cdc47d5d2806dc832db94f6a3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 11:44:30 +0000 Subject: [PATCH 67/81] Backport #117407 to 26.8: Fix TOTP bypass in MySQL protocol authentication --- src/Access/Authentication.cpp | 23 +++ src/Access/Authentication.h | 14 ++ src/Access/IAccessStorage.cpp | 9 + src/Server/MySQLHandler.cpp | 24 ++- .../test_totp_auth/config/mysql.xml | 3 + .../test_totp_auth/config/users.xml | 20 ++ tests/integration/test_totp_auth/test_totp.py | 171 ++++++++++++++++++ 7 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_totp_auth/config/mysql.xml diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index fa33d8eca45bf..055109dc596ba 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -132,6 +132,13 @@ namespace const MySQLNative41Credentials * mysql_credentials, const AuthenticationData & authentication_method) { + /// The Native41 auth response is a hash of the password alone, so it cannot carry a one-time password. + /// Accepting it would let the client skip the second factor. `MySQLHandler` switches such users + /// to the `Sha256Password` plugin, which transmits the actual password string with the one-time + /// password appended, verified via the `BasicCredentials` path. + if (authentication_method.getOneTimePassword()) + return false; + switch (authentication_method.getType()) { case AuthenticationType::PLAINTEXT_PASSWORD: @@ -416,6 +423,22 @@ Authentication::CredentialsCheckResult Authentication::areCredentialsValid( return CredentialsCheckResult::Fail; } +bool Authentication::mysqlNative41CannotVerify(const AuthenticationData & authentication_method) +{ + if (authentication_method.getOneTimePassword()) + return true; + + switch (authentication_method.getType()) + { + case AuthenticationType::SHA256_PASSWORD: + case AuthenticationType::SCRAM_SHA256_PASSWORD: + case AuthenticationType::BCRYPT_PASSWORD: + return true; + default: + return false; + } +} + bool Authentication::consumeOneTimePassword(const Credentials & credentials, const AuthenticationData & authentication_method) { const auto & otp_secret = authentication_method.getOneTimePassword(); diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index e8ed319ec9b59..5e432733caab4 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -44,6 +44,20 @@ struct Authentication /// (e.g. a wrong password combined with a valid code) cannot consume the code. static bool consumeOneTimePassword(const Credentials & credentials, const AuthenticationData & authentication_method); + /** The `mysql_native_password` auth response (`MySQLNative41Credentials`) is a hash of the password alone, so it can be + * verified only against `plaintext_password` and `double_sha1_password` methods, and it cannot carry a one-time password. + * Accepting it for a user having any method for which this returns true would be wrong in two ways: + * - a method with a second factor would be accepted without the one-time password, bypassing it; + * - `IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same password is accepted by + * several methods, the session is limited to the intersection of their `GRANTS` and expires at the earliest of + * their `VALID UNTIL`. A `sha256_password`, `scram_sha256_password` or `bcrypt_password` method cannot be matched + * by the Native41 response, so it would silently drop out of that combination, and the login would regain + * the broader rights or lifetime of the other method. + * Methods verified against an external system never take part in that combination and are not password hashes + * that a plaintext password could check better, so they do not count. + */ + static bool mysqlNative41CannotVerify(const AuthenticationData & authentication_method); + // A signaling class used to communicate requirements for credentials. template class Require : public Exception diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 10aa91bc2093e..c1c8ca1d4e5b6 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -655,6 +656,14 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); + /// `MySQLHandler` chooses the `mysql_native_password` plugin from an earlier snapshot of the user, and the user + /// can be altered while the client is answering. Re-check on the snapshot used for the authentication: if any + /// method cannot be enforced by the Native41 auth response, reject it instead of letting that method silently + /// drop out of the checks below. See `Authentication::mysqlNative41CannotVerify`. + if (typeid_cast(&credentials) + && std::ranges::any_of(user->authentication_methods, Authentication::mysqlNative41CannotVerify)) + throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials"); + bool skipped_not_allowed_authentication_methods = false; bool need_second_factor = false; diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 13935344a3140..daeb3f32f8af4 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -3,7 +3,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -763,16 +766,19 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl { try { - const auto user_authentication_types = session->getAuthenticationTypesOrLogInFailure(user_name); - - for (const auto user_authentication_type : user_authentication_types) + /// Logs a login failure and throws if there is no such user. + session->getAuthenticationTypesOrLogInFailure(user_name); + + /// For compatibility with the JavaScript MySQL client, the `mysql_native_password` plugin (Native41) is used + /// when it can fully authenticate the user. Otherwise the `sha256_password` plugin is used: it transmits the + /// actual password string, which is then checked via the `BasicCredentials` path, the same as for the native + /// and HTTP protocols. See `Authentication::mysqlNative41CannotVerify`. + /// The user can be altered while the client is answering, so this choice is only a hint: `IAccessStorage::authenticateImpl` + /// re-checks it against the snapshot of the user it authenticates with and rejects Native41 credentials if needed. + const auto user = session->globalContext()->getAccessControl().read(user_name); + if (std::ranges::any_of(user->authentication_methods, DB::Authentication::mysqlNative41CannotVerify)) { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible - // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. - if (user_authentication_type == DB::AuthenticationType::SHA256_PASSWORD) - { - authPluginSSL(); - } + authPluginSSL(); } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; diff --git a/tests/integration/test_totp_auth/config/mysql.xml b/tests/integration/test_totp_auth/config/mysql.xml new file mode 100644 index 0000000000000..782fbb025b115 --- /dev/null +++ b/tests/integration/test_totp_auth/config/mysql.xml @@ -0,0 +1,3 @@ + + 9004 + diff --git a/tests/integration/test_totp_auth/config/users.xml b/tests/integration/test_totp_auth/config/users.xml index 33441ee8cea8b..76101d677258d 100644 --- a/tests/integration/test_totp_auth/config/users.xml +++ b/tests/integration/test_totp_auth/config/users.xml @@ -76,6 +76,26 @@ + + aa+bb + + JBSWY3DPEHPK3PXPJBSWY3DPEHPK3PXP + 60 + 9 + SHA256 + + + + + e395796d6546b1b65db9d665cd43f0e858dd4303 + + GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ + 60 + 9 + SHA256 + + + pw diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 4450173a5cd9f..996e4813787a2 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -3,10 +3,12 @@ import hmac import os import re +import socket import struct import time import xml.etree.ElementTree as ET +import pymysql import pytest from helpers.cluster import ClickHouseCluster @@ -22,10 +24,15 @@ INTERACTIVE_USERS = [f"totuser_interactive_{i}" for i in range(3)] NO_PASSWORD_USERS = [f"totuser_no_password_{i}" for i in range(2)] EMPTY_PASSWORD_USERS = [f"totuser_empty_password_{i}" for i in range(2)] +MYSQL_USERS = { + "totuser_mysql_plaintext": "aa+bb", + "totuser_mysql_double_sha1": "abacaba", +} cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", + main_configs=["config/mysql.xml"], user_configs=["config/users.xml"], stay_alive=True, ) @@ -204,6 +211,170 @@ def test_interactive_totp_authentication(started_cluster): c.expect(expected_error) +def mysql_connect(user, password): + return pymysql.connections.Connection( + host=node.ip_address, + user=user, + password=password, + database="default", + port=9004, + ) + + +def test_mysql_protocol_requires_totp(started_cluster): + """The MySQL protocol must enforce TOTP: the `mysql_native_password` auth response is a hash + of the password alone and cannot carry a one-time password, so the server switches such + users to the `sha256_password` plugin and the client appends the TOTP to the password. + """ + + for user, password in MYSQL_USERS.items(): + # The correct password alone must not authenticate: the second factor is required. + # The message is intentionally generic; only the error code tells the reason. + with pytest.raises(pymysql.err.MySQLError) as exc_info: + mysql_connect(user, password) + assert exc_info.value.args[0] == 767 # REQUIRED_SECOND_FACTOR + + # A wrong one-time password must not authenticate. + with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): + mysql_connect(user, password + "+000000000") + + conn = mysql_connect(user, f"{password}+{get_otp(user)}") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == ((user,),) + finally: + conn.close() + + +def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): + """`IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same + password is accepted by several methods, the session expires at the earliest of their + `VALID UNTIL` and is limited to the intersection of their `GRANTS`. The `mysql_native_password` + auth response can only be re-checked against `plaintext_password` and `double_sha1_password` + methods, so a `scram_sha256_password` or `bcrypt_password` method that narrows the session + would silently drop out of that combination. Such users are switched to the `sha256_password` + plugin, which transmits the actual password, so every method takes part in the check. + """ + + for user, other_method in ( + ("mysql_double_sha1_and_scram", "scram_sha256_password"), + ("mysql_double_sha1_and_bcrypt", "bcrypt_password"), + ): + node.query( + f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'pw', " + f"{other_method} BY 'pw' VALID UNTIL '2000-01-01 00:00:00'" + ) + try: + # The native protocol rejects the shared password as expired ... + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + "SELECT 1", user=user, password="pw" + ) + # ... and so must the MySQL protocol. + with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): + mysql_connect(user, "pw") + + # Without the narrowing method the user is served by `mysql_native_password` again. + node.query( + f"ALTER USER {user} IDENTIFIED WITH double_sha1_password BY 'pw'" + ) + conn = mysql_connect(user, "pw") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == ((user,),) + finally: + conn.close() + finally: + node.query(f"DROP USER IF EXISTS {user}") + + # A user with only such a method could not log in over MySQL at all before: now `sha256_password` serves it. + node.query( + "CREATE USER mysql_scram_only IDENTIFIED WITH scram_sha256_password BY 'pw'" + ) + try: + conn = mysql_connect("mysql_scram_only", "pw") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == (("mysql_scram_only",),) + finally: + conn.close() + finally: + node.query("DROP USER IF EXISTS mysql_scram_only") + + +def mysql_read_packet(sock): + def read_exactly(n): + data = b"" + while len(data) < n: + chunk = sock.recv(n - len(data)) + assert chunk, "connection closed by the server" + data += chunk + return data + + header = read_exactly(4) + payload_size = int.from_bytes(header[:3], "little") + return header[3], read_exactly(payload_size) + + +def mysql_write_packet(sock, sequence_id, payload): + sock.sendall(len(payload).to_bytes(3, "little") + bytes([sequence_id]) + payload) + + +def mysql_native_password_scramble(password, salt): + password_sha1 = hashlib.sha1(password.encode()).digest() + digest = hashlib.sha1(salt + hashlib.sha1(password_sha1).digest()).digest() + return bytes(a ^ b for a, b in zip(password_sha1, digest)) + + +def test_mysql_protocol_user_altered_during_authentication(started_cluster): + """The plugin is chosen from the user before the client answers, and the user can be altered meanwhile. + The `mysql_native_password` response must still be rejected if the user authenticated against has + a method that this plugin cannot enforce. The client is paused deterministically: it asks for + `sha256_password`, so the server sends an `AuthSwitchRequest` to `mysql_native_password` and waits. + """ + + user = "mysql_altered_during_auth" + node.query(f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'old'") + try: + with socket.create_connection((node.ip_address, 9004), timeout=30) as sock: + mysql_read_packet(sock) # Initial handshake. + + CLIENT_PROTOCOL_41 = 0x200 + CLIENT_SECURE_CONNECTION = 0x8000 + CLIENT_PLUGIN_AUTH = 0x80000 + capabilities = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH + handshake_response = ( + struct.pack(" Date: Wed, 21 Oct 2026 13:29:15 +0000 Subject: [PATCH 68/81] Backport #127139 to 26.8: Fix a crash with estimateCompressionRatio --- .../CompressionCodecDoubleDelta.cpp | 24 +++++++++++-------- src/Compression/CompressionCodecGorilla.cpp | 17 +++++++++---- .../04812_codec_chain_reserve_overflow.sql | 22 ++++++++--------- ...te_compression_ratio_large_block.reference | 2 ++ ...estimate_compression_ratio_large_block.sql | 5 ++++ 5 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference create mode 100644 tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index e69edf428648a..94e9c60b2e8b2 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -266,17 +266,15 @@ UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) return items_count_size + data_bytes_size + first_delta_bytes_size; } -UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) +UInt64 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { - const UInt32 items_count = uncompressed_size / data_bytes_size; + const UInt64 items_count = uncompressed_size / data_bytes_size; const auto double_delta_write_spec = getDeltaMaxWriteSpecByteSize(data_bytes_size); - const UInt32 max_item_size_bits = double_delta_write_spec.prefix_bits + double_delta_write_spec.data_bits; + const UInt64 max_item_size_bits = double_delta_write_spec.prefix_bits + double_delta_write_spec.data_bits; - // + 8 is to round up to next byte. - auto result = (items_count * max_item_size_bits + 7) / 8; - - return result; + // + 7 is to round up to next byte. + return (items_count * max_item_size_bits + 7) / 8; } template @@ -326,7 +324,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) prev_value = curr_value; } - BitWriter writer(dest, getCompressedDataSize(sizeof(ValueType), source_size - sizeof(ValueType)*2)); + BitWriter writer(dest, getCompressedDataSize(sizeof(ValueType), static_cast(source_end - source))); int item = 2; for (; source < source_end; source += sizeof(ValueType), ++item) @@ -486,12 +484,18 @@ void CompressionCodecDoubleDelta::updateHash(SipHash & hash) const UInt32 CompressionCodecDoubleDelta::getMaxCompressedDataSize(UInt32 uncompressed_size) const { - const auto result = 2 // common header + const UInt64 result = 2 // common header + data_bytes_size // max bytes skipped if source is not properly aligned. + getCompressedHeaderSize(data_bytes_size) // data-specific header + getCompressedDataSize(data_bytes_size, uncompressed_size); - return result; + /// The block header stores the compressed block size as UInt32. + if (result > std::numeric_limits::max() - getHeaderSize()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Cannot compress with DoubleDelta codec, block of {} bytes is too large: worst-case compressed size {} does not fit into 32 bits", + uncompressed_size, result); + + return static_cast(result); } UInt32 CompressionCodecDoubleDelta::doCompressData(const char * source, UInt32 source_size, char * dest) const diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 8c937ac5b4875..b605a9d920c4a 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -13,6 +13,7 @@ #include #include +#include namespace DB @@ -163,9 +164,9 @@ UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) return items_count_size + data_bytes_size; } -UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) +UInt64 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { - const UInt32 items_count = uncompressed_size / data_bytes_size; + const UInt64 items_count = uncompressed_size / data_bytes_size; /// Not static: data_bytes_size is an argument, so a static keeps the first caller's width. const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); @@ -174,7 +175,7 @@ UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) // worst case (for 32-bit value): // 11 + 5 bits of leading zeroes bit-size + 5 bits of data bit-size + non-zero data bits. - const UInt32 max_item_size_bits = 2 + LEADING_ZEROES_BIT_LENGTH + DATA_BIT_LENGTH + data_bytes_size * 8; + const UInt64 max_item_size_bits = 2 + LEADING_ZEROES_BIT_LENGTH + DATA_BIT_LENGTH + data_bytes_size * 8; // + 8 is to round up to next byte. return (items_count * max_item_size_bits + 8) / 8; @@ -383,12 +384,18 @@ void CompressionCodecGorilla::updateHash(SipHash & hash) const UInt32 CompressionCodecGorilla::getMaxCompressedDataSize(UInt32 uncompressed_size) const { - const auto result = 2 // common header + const UInt64 result = 2 // common header + data_bytes_size // max bytes skipped if source is not properly aligned. + getCompressedHeaderSize(data_bytes_size) // data-specific header + getCompressedDataSize(data_bytes_size, uncompressed_size); - return result; + /// The block header stores the compressed block size as UInt32. + if (result > std::numeric_limits::max() - getHeaderSize()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Cannot compress with Gorilla codec, block of {} bytes is too large: worst-case compressed size {} does not fit into 32 bits", + uncompressed_size, result); + + return static_cast(result); } UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 source_size, char * dest) const diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql index f047a1d4dc1af..c61a43f96ce5d 100644 --- a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql @@ -1,7 +1,7 @@ -- Tags: no-fasttest, no-parallel -- Tag no-fasttest: a codec chain long enough to overflow a 32-bit reserve needs a second or two. --- Tag no-parallel: the 104-stage arm below allocates just under 4 GiB against a per-user memory --- limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. +-- Tag no-parallel: the longest accepted chains below reserve just under 4 GiB against a per-user +-- memory limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. -- Random settings limits: min_columns_to_activate_adaptive_write_buffer=(3, None) -- CompressionCodecMultiple compounded each stage's reserved size in a UInt32 without checking for @@ -63,9 +63,9 @@ CREATE TABLE t_codec_chain_gorilla_wide (d Float64 CODEC(Gorilla)) ENGINE = Merg INSERT INTO t_codec_chain_gorilla_wide SELECT number / 7 FROM numbers(4); SELECT count() FROM t_codec_chain_gorilla_wide; --- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 29 stages is the +-- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 36 stages is the -- longest that still fits; it must keep working too. -CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla_max SELECT number, number / 7 FROM numbers(5000); @@ -74,35 +74,35 @@ SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_gorilla_max; -- The reserve is compounded the same way whatever the codec, so the boundary below is Gorilla's own -- and one stage past it is rejected, as for FPC above. The three arms that follow pin the sibling -- formulas, so that rejection cannot silently narrow to FPC alone. -CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } SELECT count() FROM t_codec_chain_gorilla_over; --- The same boundary for a Float64 Gorilla column: 48 stages fit, 49 do not. A chain of one width +-- The same boundary for a Float64 Gorilla column: 58 stages fit, 59 do not. A chain of one width -- must be judged on its own width, so these two keep their verdicts however the Float32 arms above -- left the process. -CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla64_max SELECT number, number / 7 FROM numbers(2500); SELECT count(), countIf(d != i / 7) FROM t_codec_chain_gorilla64_max; -CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla64_over SELECT number, number / 7 FROM numbers(2500); -- { serverError CANNOT_COMPRESS } SELECT count() FROM t_codec_chain_gorilla64_over; --- DoubleDelta reserves more per stage still: 13 is the longest chain that fits, 14 is rejected. -CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i +-- DoubleDelta reserves more per stage still: 15 is the longest chain that fits, 16 is rejected. +CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_dd_max SELECT number, number / 7 FROM numbers(5000); SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_dd_max; -CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_dd_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } diff --git a/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference new file mode 100644 index 0000000000000..1c2c5f0ea97f1 --- /dev/null +++ b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference @@ -0,0 +1,2 @@ +16 +8 diff --git a/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql new file mode 100644 index 0000000000000..014d9f2f8ce0e --- /dev/null +++ b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql @@ -0,0 +1,5 @@ +-- Tags: no-sanitizers +-- no-sanitizers: too slow + +SELECT round(estimateCompressionRatio('DoubleDelta', 232160396)(toUInt16(number)), 2) FROM numbers(116080198) SETTINGS max_threads = 1, max_rows_to_read = 0; +SELECT round(estimateCompressionRatio('Gorilla', 252645136)(toUInt8(0)), 2) FROM numbers(252645136) SETTINGS max_threads = 1, max_rows_to_read = 0; From 912b9c35b21cb861d487b3aa32c0146851f9ae64 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 15:17:15 +0000 Subject: [PATCH 69/81] Backport #127043 to 26.8: Improve glue catalog cache --- src/Common/CurrentMetrics.cpp | 2 + src/Databases/DataLake/GlueCatalog.cpp | 54 +++++++++++++------ src/Databases/DataLake/GlueCatalog.h | 13 ++++- tests/integration/test_database_glue/test.py | 56 ++++++++++++++++++++ 4 files changed, 108 insertions(+), 17 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 15e08b2903a9f..8be37f19d9536 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -369,6 +369,8 @@ M(AsyncInsertCacheSize, "Number of async insert hash id in cache") \ M(IcebergMetadataFilesCacheBytes, "Size of the Iceberg metadata cache in bytes") \ M(IcebergMetadataFilesCacheFiles, "Number of cached files in the Iceberg metadata cache") \ + M(GlueCatalogSchemaCacheBytes, "Size of the cache of current Iceberg schema field types in Glue catalogs in bytes") \ + M(GlueCatalogSchemaCacheEntries, "Number of entries in the cache of current Iceberg schema field types in Glue catalogs") \ M(PaimonMetadataFilesCacheBytes, "Size of the Paimon metadata cache in bytes") \ M(PaimonMetadataFilesCacheFiles, "Number of cached files in the Paimon metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index a2817627e33da..6a0258f305406 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -85,8 +86,8 @@ namespace DB::ServerSetting namespace CurrentMetrics { - extern const Metric MarkCacheBytes; - extern const Metric MarkCacheFiles; + extern const Metric GlueCatalogSchemaCacheBytes; + extern const Metric GlueCatalogSchemaCacheEntries; } namespace DataLake @@ -119,7 +120,7 @@ GlueCatalog::GlueCatalog( , region(settings_.region) , settings(settings_) , table_engine_definition(table_engine_definition_) - , metadata_objects(CurrentMetrics::MarkCacheBytes, CurrentMetrics::MarkCacheFiles, 1024) + , current_schema_field_types(CurrentMetrics::GlueCatalogSchemaCacheBytes, CurrentMetrics::GlueCatalogSchemaCacheEntries, 16_MiB) { DB::S3::CredentialsConfiguration creds_config; creds_config.use_environment_credentials = true; @@ -515,17 +516,16 @@ String GlueCatalog::getActualTimestampType(const String & column_name, const Tab const String & metadata_uri = table_specific_properties->iceberg_metadata_file_location; - if (!metadata_objects.get(metadata_uri)) + auto [field_types, _] = current_schema_field_types.getOrSet(metadata_uri, [&] { auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(metadata_uri, table_metadata); auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(metadata_uri); auto metadata_object = DB::Iceberg::getMetadataJSONObject( metadata_path, object_storage, nullptr, getContext(), log, compression_method, std::nullopt); - metadata_objects.set(metadata_uri, std::make_shared(metadata_object)); - } + return std::make_shared(getCurrentSchemaFieldTypes(metadata_object)); + }); - auto metadata_object = *metadata_objects.get(metadata_uri); - return resolveTimestampTypeFromMetadata(metadata_object, column_name, glue_column_type); + return resolveTimestampType(*field_types, column_name, glue_column_type); } String GlueCatalog::resolveTimestampTypeFromMetadata( @@ -533,26 +533,48 @@ String GlueCatalog::resolveTimestampTypeFromMetadata( const String & column_name, const String & glue_column_type) { + return resolveTimestampType(getCurrentSchemaFieldTypes(metadata_object), column_name, glue_column_type); +} + +GlueCatalog::FieldTypes GlueCatalog::getCurrentSchemaFieldTypes(const Poco::JSON::Object::Ptr & metadata_object) +{ + FieldTypes result; auto current_schema_id = metadata_object->getValue("current-schema-id"); auto schemas = metadata_object->getArray(DB::Iceberg::f_schemas); for (size_t i = 0; i < schemas->size(); ++i) { auto schema = schemas->getObject(static_cast(i)); - if (schema->getValue("schema-id") == current_schema_id) + if (schema->getValue("schema-id") != current_schema_id) + continue; + + auto fields = schema->getArray(DB::Iceberg::f_fields); + for (size_t j = 0; j < fields->size(); ++j) { - auto fields = schema->getArray(DB::Iceberg::f_fields); - for (size_t j = 0; j < fields->size(); ++j) - { - auto field = fields->getObject(static_cast(j)); - if (field->getValue(DB::Iceberg::f_name) == column_name) - return field->getValue(DB::Iceberg::f_type); - } + auto field = fields->getObject(static_cast(j)); + auto type = field->get(DB::Iceberg::f_type); + if (type.isString()) + result.emplace(field->getValue(DB::Iceberg::f_name), type.extract()); } } + return result; +} + +String GlueCatalog::resolveTimestampType(const FieldTypes & field_types, const String & column_name, const String & glue_column_type) +{ + if (auto it = field_types.find(column_name); it != field_types.end()) + return it->second; return glue_column_type == "timestamp_nano" ? "timestamp_ns" : "timestamp"; } +size_t GlueCatalog::FieldTypesWeightFunction::operator()(const FieldTypes & field_types) const +{ + size_t result = 0; + for (const auto & [name, type] : field_types) + result += sizeof(FieldTypes::value_type) + name.size() + type.size(); + return result; +} + GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const { DB::ASTStorage * storage = table_engine_definition->as(); diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 5007e5e939744..5122a95303515 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -96,7 +96,18 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & column_name, const String & glue_column_type); + using FieldTypes = std::unordered_map; + + static FieldTypes getCurrentSchemaFieldTypes(const Poco::JSON::Object::Ptr & metadata_object); + + static String resolveTimestampType(const FieldTypes & field_types, const String & column_name, const String & glue_column_type); + private: + struct FieldTypesWeightFunction + { + size_t operator()(const FieldTypes & field_types) const; + }; + std::unique_ptr glue_client; const LoggerPtr log; std::shared_ptr credentials_provider; @@ -125,7 +136,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext ObjectStorageWithPath createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const; - mutable DB::CacheBase metadata_objects; + mutable DB::CacheBase, FieldTypesWeightFunction> current_schema_field_types; }; } diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 50098dc9bae6d..f8e45e9bbe921 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -812,6 +812,62 @@ def test_timestamps(started_cluster): assert node.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`") == "2024-01-01 12:00:00.000000\t2024-01-01 12:00:00.000000\n" +def test_timestamp_types_cache_does_not_keep_metadata(started_cluster): + node = started_cluster.instances["node1"] + + test_ref = f"test_ts_cache_{uuid.uuid4().hex}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + database_name = f"db_{test_ref}" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + + schema = Schema( + NestedField(field_id=1, name="timestamp", field_type=TimestampType(), required=False), + NestedField(field_id=2, name="timestamptz", field_type=TimestamptzType(), required=False), + ) + table = create_table(catalog, root_namespace, table_name, schema, dir=test_ref) + + create_clickhouse_glue_database(started_cluster, node, database_name) + + def cache_metrics(): + return [ + int(value) + for value in node.query( + "SELECT value FROM system.metrics " + "WHERE metric IN ('GlueCatalogSchemaCacheBytes', 'GlueCatalogSchemaCacheEntries') ORDER BY metric" + ).split() + ] + + bytes_before, entries_before = cache_metrics() + + row = pa.Table.from_pylist( + [{"timestamp": datetime(2024, 1, 1, 12), "timestamptz": datetime(2024, 1, 1, 12, tzinfo=pytz.UTC)}] + ) + versions = 30 + for _ in range(versions): + table.append(row) + description = node.query(f"DESCRIBE TABLE {database_name}.`{root_namespace}.{table_name}`") + assert [line.split("\t")[1] for line in description.splitlines()] == [ + "Nullable(DateTime64(6))", + "Nullable(DateTime64(6, \\'UTC\\'))", + ] + + bytes_after, entries_after = cache_metrics() + assert entries_after - entries_before == versions + + metadata_size = int( + node.query( + f"SELECT max(_size) FROM s3('http://minio1:9001/warehouse-glue/{test_ref}/metadata/*.metadata.json', " + f"'{minio_access_key}', '{minio_secret_key}', 'One')" + ) + ) + assert bytes_after - bytes_before < metadata_size + + node.query(f"DROP DATABASE {database_name}") + + def test_insert(started_cluster): node = started_cluster.instances["node1"] From d4b3a312151cc16172584dde1f9d991164a2cf00 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 15:27:59 +0000 Subject: [PATCH 70/81] Backport #125823 to 26.8: Return `REQUIRED_SECOND_FACTOR` regardless of the password when the TOTP code is missing --- .../configuration/settings/settings-users.mdx | 1 + programs/client/Client.cpp | 14 +++- src/Access/Authentication.cpp | 77 +++++++++++-------- src/Access/IAccessStorage.cpp | 2 + src/Server/HTTP/exceptionCodeToHTTPStatus.cpp | 3 +- tests/integration/test_totp_auth/test_totp.py | 41 +++++++++- 6 files changed, 100 insertions(+), 38 deletions(-) diff --git a/docs/concepts/features/configuration/settings/settings-users.mdx b/docs/concepts/features/configuration/settings/settings-users.mdx index 0b7caa2651214..b39f2d3fba25f 100644 --- a/docs/concepts/features/configuration/settings/settings-users.mdx +++ b/docs/concepts/features/configuration/settings/settings-users.mdx @@ -123,6 +123,7 @@ It's not yet supported in SQL-driven Access Control. To authenticate using TOTP, users must provide a primary password along with a one-time password generated by their TOTP application via the `--one-time-password` command line option or concatenated to main password together with a '+' character. For example if the primary password is `some_password` and the generated TOTP code is `345123`, user may specify `--password some_password+345123` or `--password some_password --one-time-password 345123` when connecting to ClickHouse. If no password specified `clickhouse-client` will prompt for it interactively. +When no code is supplied for an account with TOTP, the server replies with the error `REQUIRED_SECOND_FACTOR` whether or not the password is correct, and `clickhouse-client` prompts for the code; the password is verified only together with the code. Each code is accepted at most once, as required by [RFC 6238](https://datatracker.ietf.org/doc/html/rfc6238#section-5.2), so a user can successfully authenticate at most once per `period`. Keep this in mind for clients that authenticate on every request or connection (for example, scripts using the HTTP interface): each authentication needs a fresh code. diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index b4d1d5db053c9..9d9532335fd46 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -443,6 +443,17 @@ try if (should_ask_2fa) { + /// The server asks for the second factor before it checks the password, so the first attempt + /// without any password ends here too. Ask for the password alone first: it may already contain + /// the code (`password+code`); otherwise the server asks again and the code is prompted. + if (!asked_password && !config().has("password") && !config().getBool("ask-password", false)) + { + asked_password = true; + config().setBool("ask-password", true); + preserve_announced_endpoint_for_retry = true; + continue; + } + asked_2fa = true; if (!connection_parameters.password.empty()) config().setString("password", connection_parameters.password); @@ -923,7 +934,8 @@ void Client::connect() } /// This problem can't be fixed with reconnection so it is not attempted - if (e.code() == ErrorCodes::AUTHENTICATION_FAILED || e.code() == ErrorCodes::REQUIRED_PASSWORD) + if (e.code() == ErrorCodes::AUTHENTICATION_FAILED || e.code() == ErrorCodes::REQUIRED_PASSWORD + || e.code() == ErrorCodes::REQUIRED_SECOND_FACTOR) throw; if (attempted_address_index == hosts_and_ports.size() - 1) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index fa33d8eca45bf..2669b7c3fccf2 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -176,80 +176,60 @@ namespace return {password_with_otp.substr(0, separator_pos), password_with_otp.substr(separator_pos + 1)}; } - Authentication::CredentialsCheckResult checkBasicAuthentication( - const BasicCredentials * basic_credentials, + /// Verifies the primary credential: the password without the one-time password part. + bool checkPrimaryCredential( + std::string_view password, + const BasicCredentials & basic_credentials, const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, const ClientInfo & client_info, SettingsChanges & settings) { - const auto & provided_password = basic_credentials->getPassword(); - const auto & otp_secret = authentication_method.getOneTimePassword(); - auto [password, one_time_password] = splitOneTimePasswordAndPassword(provided_password, authentication_method); - Authentication::CredentialsCheckResult on_success = Authentication::CredentialsCheckResult::Success; - if (otp_secret) - { - /// Only a pure check here: the code is marked as used by `consumeOneTimePassword` after - /// the whole authentication succeeds, so a failed attempt cannot burn a valid code. - if (one_time_password.empty()) - on_success = Authentication::CredentialsCheckResult::NeedSecondFactor; - else if (!checkOneTimePassword(one_time_password, *otp_secret)) - return Authentication::CredentialsCheckResult::Fail; - } - switch (authentication_method.getType()) { case AuthenticationType::NO_PASSWORD: { - return on_success; // N.B. even if the password is not empty! + return true; // N.B. even if the password is not empty! } case AuthenticationType::PLAINTEXT_PASSWORD: { const auto & password_plaintext = authentication_method.getPasswordHashBinary(); - return constantTimeEquals(Util::stringToDigest(password), password_plaintext) ? on_success : Authentication::CredentialsCheckResult::Fail; + return constantTimeEquals(Util::stringToDigest(password), password_plaintext); } case AuthenticationType::SHA256_PASSWORD: { const auto & password_sha256 = authentication_method.getPasswordHashBinary(); const auto & salt = authentication_method.getSalt(); String salted_password = String(password).append(salt); - return Util::encodeSHA256(salted_password) == password_sha256 ? on_success : Authentication::CredentialsCheckResult::Fail; + return Util::encodeSHA256(salted_password) == password_sha256; } case AuthenticationType::SCRAM_SHA256_PASSWORD: { const auto & password_scram_sha256 = authentication_method.getPasswordHashBinary(); const auto & salt = authentication_method.getSalt(); auto digest = Util::encodeScramSHA256(password, salt); - return digest == password_scram_sha256 ? on_success : Authentication::CredentialsCheckResult::Fail; + return digest == password_scram_sha256; } case AuthenticationType::DOUBLE_SHA1_PASSWORD: { const auto & password_double_sha1 = authentication_method.getPasswordHashBinary(); - return Util::encodeDoubleSHA1(password) == password_double_sha1 ? on_success : Authentication::CredentialsCheckResult::Fail; + return Util::encodeDoubleSHA1(password) == password_double_sha1; } case AuthenticationType::LDAP: { - if (otp_secret) - /// One-time password supported only with password-based authentication methods. - return Authentication::CredentialsCheckResult::Fail; - return external_authenticators.checkLDAPCredentials(authentication_method.getLDAPServerName(), *basic_credentials) ? - on_success : Authentication::CredentialsCheckResult::Fail; + return external_authenticators.checkLDAPCredentials(authentication_method.getLDAPServerName(), basic_credentials); } case AuthenticationType::BCRYPT_PASSWORD: { const auto & password_bcrypt = authentication_method.getPasswordHashBinary(); - return Util::checkPasswordBcrypt(password, password_bcrypt) ? on_success : Authentication::CredentialsCheckResult::Fail; + return Util::checkPasswordBcrypt(password, password_bcrypt); } case AuthenticationType::HTTP: { - if (otp_secret) - /// One-time password supported only with password-based authentication methods. - return Authentication::CredentialsCheckResult::Fail; if (authentication_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC) { return external_authenticators.checkHTTPBasicCredentials( - authentication_method.getHTTPAuthenticationServerName(), *basic_credentials, client_info, settings) ? - on_success : Authentication::CredentialsCheckResult::Fail; + authentication_method.getHTTPAuthenticationServerName(), basic_credentials, client_info, settings); } break; } @@ -257,7 +237,38 @@ namespace break; } - return Authentication::CredentialsCheckResult::Fail; + return false; + } + + Authentication::CredentialsCheckResult checkBasicAuthentication( + const BasicCredentials * basic_credentials, + const AuthenticationData & authentication_method, + const ExternalAuthenticators & external_authenticators, + const ClientInfo & client_info, + SettingsChanges & settings) + { + const auto & otp_secret = authentication_method.getOneTimePassword(); + auto [password, one_time_password] = splitOneTimePasswordAndPassword(basic_credentials->getPassword(), authentication_method); + + const auto auth_type = authentication_method.getType(); + if (otp_secret && (auth_type == AuthenticationType::LDAP || auth_type == AuthenticationType::HTTP)) + /// One-time password supported only with password-based authentication methods. + return Authentication::CredentialsCheckResult::Fail; + + /// Decided before the primary credential is looked at, so that the response (and its timing) is the same + /// for a right and a wrong password: otherwise an attacker could confirm a guessed password without the code. + if (otp_secret && one_time_password.empty()) + return Authentication::CredentialsCheckResult::NeedSecondFactor; + + const bool primary_ok = checkPrimaryCredential(password, *basic_credentials, authentication_method, external_authenticators, client_info, settings); + if (!otp_secret) + return primary_ok ? Authentication::CredentialsCheckResult::Success : Authentication::CredentialsCheckResult::Fail; + + /// Only a pure check here: the code is marked as used by `consumeOneTimePassword` after + /// the whole authentication succeeds, so a failed attempt cannot burn a valid code. + /// Both factors are evaluated unconditionally, so that the timing does not tell which one failed. + const bool otp_ok = checkOneTimePassword(one_time_password, *otp_secret); + return (primary_ok && otp_ok) ? Authentication::CredentialsCheckResult::Success : Authentication::CredentialsCheckResult::Fail; } #if USE_SSL diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 10aa91bc2093e..aea969c3c40d3 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -690,6 +690,8 @@ std::optional IAccessStorage::authenticateImpl( "check allow_no_password and allow_plaintext_password settings in the server configuration"); } + /// Reported for a missing code before the password is checked (see `checkBasicAuthentication`), + /// so this error does not confirm the password. if (need_second_factor) throw Exception(ErrorCodes::REQUIRED_SECOND_FACTOR, "Authentication requires second factor"); throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials"); diff --git a/src/Server/HTTP/exceptionCodeToHTTPStatus.cpp b/src/Server/HTTP/exceptionCodeToHTTPStatus.cpp index 757b096127e2f..901a9b8178370 100644 --- a/src/Server/HTTP/exceptionCodeToHTTPStatus.cpp +++ b/src/Server/HTTP/exceptionCodeToHTTPStatus.cpp @@ -60,6 +60,7 @@ namespace ErrorCodes extern const int WRONG_PASSWORD; extern const int REQUIRED_PASSWORD; extern const int AUTHENTICATION_FAILED; + extern const int REQUIRED_SECOND_FACTOR; extern const int SET_NON_GRANTED_ROLE; extern const int HTTP_LENGTH_REQUIRED; @@ -80,7 +81,7 @@ Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int exception_code } if (exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD || exception_code == ErrorCodes::AUTHENTICATION_FAILED || exception_code == ErrorCodes::SET_NON_GRANTED_ROLE - || exception_code == ErrorCodes::ACCESS_DENIED) + || exception_code == ErrorCodes::ACCESS_DENIED || exception_code == ErrorCodes::REQUIRED_SECOND_FACTOR) { return HTTPResponse::HTTP_FORBIDDEN; } diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 4450173a5cd9f..2eb8c81ebc84b 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -91,9 +91,37 @@ def test_one_time_password(started_cluster): query_text, user="totuser", password=f"aa+bb+{old_password}" ) - assert "REQUIRED_SECOND_FACTOR" in node.query_and_get_error( + # Without a code the response does not depend on the password, so the error REQUIRED_SECOND_FACTOR + # does not confirm a guessed password: a wrong or an empty password gets exactly the same error + error_without_code = node.query_and_get_error( query_text, user="totuser", password="aa+bb" ) + assert "REQUIRED_SECOND_FACTOR" in error_without_code + for password in ["wrongpwd", ""]: + assert error_without_code == node.query_and_get_error( + query_text, user="totuser", password=password + ) + + http_responses = [ + node.http_request( + "", + params={"query": query_text}, + headers={"X-ClickHouse-User": "totuser", "X-ClickHouse-Key": password}, + ) + for password in ["aa+bb", "wrongpwd"] + ] + for response in http_responses: + assert response.status_code == 403 + assert response.headers["X-ClickHouse-Exception-Code"] == "767" + assert http_responses[0].text == http_responses[1].text + + response = node.http_request( + "", + params={"query": query_text}, + headers={"X-ClickHouse-User": "totuser", "X-ClickHouse-Key": "aa+bb+000000000"}, + ) + assert response.status_code == 403 + assert response.headers["X-ClickHouse-Exception-Code"] == "516" assert "totuser42\n" == node.query( query_text, user="totuser", password=f"aa+bb+{get_otp('totuser')}" @@ -189,13 +217,17 @@ def test_interactive_totp_authentication(started_cluster): # Then enter wrong TOTP when prompted c.expect("TOTP.*:") - c.send("000000", eol="\r") + c.send("000000000", eol="\r") c.expect(expected_error) - with client(command=f"{client_command(user0)} --password aa+bb+000000") as c: + # A wrong code of the configured length (9 digits); a suffix of another length counts as no code + with client(command=f"{client_command(user0)} --password aa+bb+000000000") as c: c.expect(expected_error) + # A wrong password without a code is asked for the code like a right one, and fails only after that with client(command=f"{client_command(user0)} --password wrongpwd") as c: + c.expect("TOTP.*:") + c.send(get_otp(user0), eol="\r") c.expect(expected_error) with client( @@ -214,7 +246,10 @@ def test_one_time_only_no_password(started_cluster): assert f"{user0}42\n" == node.query(query_text, user=user0, password=get_otp(user0)) + # The client cannot know that the account has no password: it asks for the password first with client(command=f"{client_command(user0)}") as c: + c.expect("Password.*:") + c.send("", eol="\r") c.expect("TOTP.*:") c.send(get_otp(user0, offset_steps=1), eol="\r") c.expect(prompt) From bc2b39ceda27c5610db839128b4b8ba1ff4b5b95 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 16:27:32 +0000 Subject: [PATCH 71/81] Backport #126546 to 26.8: Validate malformed responses from HDFS DataNode and NameNode --- contrib/libhdfs3 | 2 +- contrib/libhdfs3-cmake/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index ceb428c52e6b4..28a9868bc1574 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit ceb428c52e6b4362a35ec18b69206d9bb94edce3 +Subproject commit 28a9868bc15749d899669e117e0c6eb9f5350923 diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 8c03f283f0627..73e5cff8a48a0 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -103,6 +103,8 @@ set(SRCS "${HDFS3_SOURCE_DIR}/client/StripedBlockUtil.cpp" "${HDFS3_SOURCE_DIR}/client/StripedInputStreamImpl.cpp" "${HDFS3_SOURCE_DIR}/client/StripedOutputStreamImpl.cpp" + "${HDFS3_SOURCE_DIR}/client/SlowMetrics.cpp" + "${HDFS3_SOURCE_DIR}/client/SlowNodeTracker.cpp" "${HDFS3_SOURCE_DIR}/client/SystemECPolicies.cpp" "${HDFS3_SOURCE_DIR}/client/dump.cpp" "${HDFS3_SOURCE_DIR}/client/erasure_coder.cpp" From 5aea966ebca8c57bee9753618e2cf6cbc68bd6e9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 16:43:06 +0000 Subject: [PATCH 72/81] Update autogenerated version to 26.8.33.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index f3356873a6869..1bc0134eb7c74 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54545) +SET(VERSION_REVISION 54546) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 33) -SET(VERSION_GITHASH dc83db6f660bb74548f3309bbbf3bf4267988b66) -SET(VERSION_DESCRIBE v26.8.33.1-lts) -SET(VERSION_STRING 26.8.33.1) +SET(VERSION_PATCH 34) +SET(VERSION_GITHASH 4ca42662d1a0175631b2f340620d5a50dd049999) +SET(VERSION_DESCRIBE v26.8.34.1-lts) +SET(VERSION_STRING 26.8.34.1) # end of autochange From bcfcc883cb3febb9ff90e4e6824fe616d2886412 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 21 Oct 2026 19:31:16 +0000 Subject: [PATCH 73/81] Adapt test_mysql_protocol_fail_close_on_unverifiable_methods to 26.8 The expired-method assertions need the fail-close for ambiguous credentials of #110144, which 26.8 does not have: here a password accepted by any method logs in, on the native protocol too. Keep the part this backport changes: a user with `double_sha1_password` and a `scram_sha256_password` or `bcrypt_password` method logs in over MySQL through the `sha256_password` plugin. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_totp_auth/test_totp.py | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 996e4813787a2..f81b454a07505 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -248,13 +248,10 @@ def test_mysql_protocol_requires_totp(started_cluster): def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): - """`IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same - password is accepted by several methods, the session expires at the earliest of their - `VALID UNTIL` and is limited to the intersection of their `GRANTS`. The `mysql_native_password` - auth response can only be re-checked against `plaintext_password` and `double_sha1_password` - methods, so a `scram_sha256_password` or `bcrypt_password` method that narrows the session - would silently drop out of that combination. Such users are switched to the `sha256_password` - plugin, which transmits the actual password, so every method takes part in the check. + """The `mysql_native_password` auth response can be re-checked only against `plaintext_password` + and `double_sha1_password` methods, so `IAccessStorage::authenticateImpl` rejects it for a user + who also has a `scram_sha256_password` or `bcrypt_password` method. Such users are switched to + the `sha256_password` plugin, which transmits the actual password, so they can still log in. """ for user, other_method in ( @@ -263,21 +260,9 @@ def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): ): node.query( f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'pw', " - f"{other_method} BY 'pw' VALID UNTIL '2000-01-01 00:00:00'" + f"{other_method} BY 'pw'" ) try: - # The native protocol rejects the shared password as expired ... - assert "AUTHENTICATION_FAILED" in node.query_and_get_error( - "SELECT 1", user=user, password="pw" - ) - # ... and so must the MySQL protocol. - with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): - mysql_connect(user, "pw") - - # Without the narrowing method the user is served by `mysql_native_password` again. - node.query( - f"ALTER USER {user} IDENTIFIED WITH double_sha1_password BY 'pw'" - ) conn = mysql_connect(user, "pw") try: cursor = conn.cursor() From bab43bd6d733ce5702cc0a912f84ebe34eee4139 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 20:20:01 +0000 Subject: [PATCH 74/81] Backport #125558 to 26.8: Do not skip the client output teardown when the terminal is gone --- src/Client/ClientBase.cpp | 31 +++-- ...6_webterminal_close_during_query.reference | 3 + .../05386_webterminal_close_during_query.sh | 119 ++++++++++++++++++ 3 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05386_webterminal_close_during_query.reference create mode 100755 tests/queries/0_stateless/05386_webterminal_close_during_query.sh diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index cd2640da724c0..5ab920a0665ff 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2188,10 +2188,28 @@ void ClientBase::onProfileEvents(Block & block) /// Flush all buffers. void ClientBase::resetOutput() { - if (need_render_progress_table && tty_buf) + auto handle_exception = [this] { - std::unique_lock lock(tty_mutex); - progress_table.clearTableOutput(*tty_buf, lock); + if (!have_error) + { + client_exception + = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); + have_error = true; + } + }; + + try + { + if (need_render_progress_table && tty_buf) + { + std::unique_lock lock(tty_mutex); + progress_table.clearTableOutput(*tty_buf, lock); + } + } + catch (...) + { + /// The terminal may be gone, e.g. a closed session of the client embedded into the server. + handle_exception(); } /// Order is important: format, compression, file @@ -2205,12 +2223,7 @@ void ClientBase::resetOutput() { /// We need to make sure we continue resetting output_format (will stop threads on parallel output) /// as well as cleaning other output related setup - if (!have_error) - { - client_exception - = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); - have_error = true; - } + handle_exception(); } output_format.reset(); diff --git a/tests/queries/0_stateless/05386_webterminal_close_during_query.reference b/tests/queries/0_stateless/05386_webterminal_close_during_query.reference new file mode 100644 index 0000000000000..dfd7d133a2b2f --- /dev/null +++ b/tests/queries/0_stateless/05386_webterminal_close_during_query.reference @@ -0,0 +1,3 @@ +closed during the query +0 +0 diff --git a/tests/queries/0_stateless/05386_webterminal_close_during_query.sh b/tests/queries/0_stateless/05386_webterminal_close_during_query.sh new file mode 100755 index 0000000000000..0d5cc13ccac4f --- /dev/null +++ b/tests/queries/0_stateless/05386_webterminal_close_during_query.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A web terminal session closed while a query is running must not leave the +# output buffers of the embedded client neither finalized nor canceled. + +MARKER="webterminal_close_${CLICKHOUSE_DATABASE}" +START=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(now())") + +python3 - "${CLICKHOUSE_HOST}" "${CLICKHOUSE_PORT_HTTP}" "${MARKER}" <<'PYEOF' +import base64 +import json +import os +import socket +import struct +import sys +import time + +host, port, marker = sys.argv[1], int(sys.argv[2]), sys.argv[3] + + +def recv_exact(sock, n): + buf = b"" + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise EOFError("connection closed") + buf += chunk + return buf + + +def send_frame(sock, opcode, payload): + mask = os.urandom(4) + masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + if len(payload) < 126: + header = bytes([0x80 | opcode, 0x80 | len(payload)]) + else: + header = bytes([0x80 | opcode, 0x80 | 126]) + struct.pack(">H", len(payload)) + sock.sendall(header + mask + masked) + + +def read_frame(sock): + header = recv_exact(sock, 2) + opcode = header[0] & 0x0F + length = header[1] & 0x7F + if length == 126: + length = struct.unpack(">H", recv_exact(sock, 2))[0] + elif length == 127: + length = struct.unpack(">Q", recv_exact(sock, 8))[0] + return opcode, recv_exact(sock, length) + + +def read_until(sock, needle, timeout): + """Accumulate the terminal output until it contains `needle`.""" + deadline = time.monotonic() + timeout + output = b"" + while needle not in output: + if time.monotonic() > deadline: + raise TimeoutError(f"{needle!r} not found in {output[-500:]!r}") + opcode, payload = read_frame(sock) + if opcode == 0x08: + raise EOFError(f"closed by the server: {payload!r}") + if opcode == 0x09: + send_frame(sock, 0x0A, payload) + elif opcode == 0x02: + output += payload + return output + + +sock = socket.create_connection((host, port), timeout=60) +key = base64.b64encode(os.urandom(16)).decode() +request = ( + f"GET /webterminal HTTP/1.1\r\n" + f"Host: {host}:{port}\r\n" + f"Origin: http://{host}:{port}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + f"Sec-WebSocket-Version: 13\r\n\r\n" +) +sock.sendall(request.encode()) +response = b"" +while b"\r\n\r\n" not in response: + response += recv_exact(sock, 1) +assert response.startswith(b"HTTP/1.1 101"), response + +send_frame(sock, 0x01, json.dumps({"type": "auth", "user": "default", "password": ""}).encode()) +read_until(sock, b":) ", 60) + +query = ( + "SELECT count() FROM numbers(100) WHERE NOT ignore(sleepEachRow(0.5)) " + f"SETTINGS max_block_size = 1, log_comment = '{marker}'\r" +) +send_frame(sock, 0x02, query.encode()) +read_until(sock, b"Progress:", 60) + +# Close the connection abruptly while the query is running. +sock.close() +print("closed during the query") +PYEOF + +# The query leaves the process list once the session of the embedded client is torn down. +for _ in {1..60}; do + running=$(${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.processes WHERE Settings['log_comment'] = '${MARKER}'") + [ "$running" = "0" ] && break + sleep 1 +done +echo "$running" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS text_log" +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.text_log + WHERE event_time >= toDateTime(${START}) + AND logger_name = 'WriteBuffer' + AND message LIKE '%neither finalized nor canceled%' + AND message LIKE '%ClientEmbeddedRunner%'" From 6e5e5074a5ae3e7c56d96f581062ce25cbf7e4cd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 00:59:27 +0000 Subject: [PATCH 75/81] Backport #126937 to 26.8: Fix shitted string offsets of sparse `Nullable(String)` columns in the native protocol --- .../Serializations/SerializationString.cpp | 34 ++++++++++++++++-- ...e_sparse_nullable_string_offsets.reference | 21 +++++++++++ ..._native_sparse_nullable_string_offsets.sql | 35 +++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.reference create mode 100644 tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.sql diff --git a/src/DataTypes/Serializations/SerializationString.cpp b/src/DataTypes/Serializations/SerializationString.cpp index 5f4d2aba16d31..7e63af7731ba5 100644 --- a/src/DataTypes/Serializations/SerializationString.cpp +++ b/src/DataTypes/Serializations/SerializationString.cpp @@ -663,6 +663,35 @@ void serializeStringSizes(const IColumn & column, WriteBuffer & ostr, UInt64 off } } +void serializeStringOffsets(const ColumnString::Offsets & offsets, WriteBuffer & ostr, size_t offset, size_t limit) +{ + const ColumnString::Offset base = offsets[offset - 1]; + if (base == 0) + { + SerializationNumber::serializeBinaryBulk(offsets, ostr, offset, limit); + return; + } + + if (limit == 0) + return; + + if constexpr (std::endian::native == std::endian::big) + { + for (size_t i = offset; i < offset + limit; ++i) + writeBinaryLittleEndian(offsets[i] - base, ostr); + } + else + { + PaddedPODArray rebased; + rebased.resize(limit); + + for (size_t i = 0; i < limit; ++i) + rebased[i] = offsets[offset + i] - base; + + ostr.write(reinterpret_cast(rebased.data()), sizeof(ColumnString::Offset) * limit); + } +} + void appendStringSizesToColumnStringOffsets(ColumnString & column_string, const UInt64 * sizes, size_t start, size_t rows) { auto & offsets = column_string.getOffsets(); @@ -746,12 +775,11 @@ void SerializationString::serializeBinaryBulkWithSizeStream( const auto & offsets = column_string.getOffsets(); /// On disk (position independent) the stream carries per-row sizes so a granule reads on its own. - /// Over the network it carries the cumulative offsets directly (like Array offsets) for exact - /// reader preallocation. + /// Over the network it carries cumulative offsets counted from the first written row, for exact reader preallocation. if (settings.position_independent_encoding) serializeStringSizes(column, *size_stream, offset, limit); else - SerializationNumber::serializeBinaryBulk(offsets, *size_stream, offset, limit); + serializeStringOffsets(offsets, *size_stream, offset, limit); settings.path.back() = Substream::Regular; auto * stream = settings.getter(settings.path); diff --git a/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.reference b/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.reference new file mode 100644 index 0000000000000..67d14e1a100e0 --- /dev/null +++ b/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.reference @@ -0,0 +1,21 @@ +Sparse +0 \N +1 \N +49 1813 +99 3663 +Sparse +0 \N +1 -0 +2 nan +3 nan +4 \N +0 \N \N +0 \N \N +1 -0 2 +1 -0 2 +2 nan 3 +2 nan 3 +3 nan 3 +3 nan 3 +4 \N \N +4 \N \N diff --git a/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.sql b/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.sql new file mode 100644 index 0000000000000..07789ecb1a70f --- /dev/null +++ b/tests/queries/0_stateless/05427_native_sparse_nullable_string_offsets.sql @@ -0,0 +1,35 @@ +-- Tags: shard +-- Random settings limits: optimize_read_in_order=(1, 1) + +-- Over the native protocol a sparse `Nullable(String)` is sent as the values from index 1, and the NULL +-- default row at index 0 may keep a non-empty nested string ('0' below); the offsets must not include it. + +DROP TABLE IF EXISTS t_sparse_int; +CREATE TABLE t_sparse_int (k UInt32, i Int64) ENGINE = MergeTree ORDER BY k +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5; +INSERT INTO t_sparse_int SELECT number, if(number % 50 = 49, number * 37, 0) FROM numbers(100); +SELECT serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_int' AND column = 'i' AND active; +SELECT k, toString(nullIf(i, 0)) FROM t_sparse_int WHERE k IN (0, 49, 99) OR k < 2 ORDER BY k; + +DROP TABLE IF EXISTS t_sparse_nullable; +CREATE TABLE t_sparse_nullable (k UInt32, c Nullable(Float64)) ENGINE = MergeTree ORDER BY k +SETTINGS nullable_serialization_version = 'allow_sparse', serialization_info_version = 'with_types', + ratio_of_defaults_for_sparse_serialization = 0.0; +INSERT INTO t_sparse_nullable VALUES (0, NULL); +INSERT INTO t_sparse_nullable VALUES (1, -0.), (2, nan), (3, nan), (4, NULL); +OPTIMIZE TABLE t_sparse_nullable FINAL; +SELECT serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_nullable' AND column = 'c' AND active; +SELECT k, toString(c) FROM t_sparse_nullable ORDER BY k; + +-- The same column read from shards over the protocol and stored. +DROP TABLE IF EXISTS t_copy; +CREATE TABLE t_copy (k UInt32, s Nullable(String)) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_copy SELECT k, toString(c) FROM remote('127.0.0.{1,2}', currentDatabase(), t_sparse_nullable) +SETTINGS prefer_localhost_replica = 0; +SELECT k, s, length(s) FROM t_copy ORDER BY k, s; + +DROP TABLE t_copy; +DROP TABLE t_sparse_nullable; +DROP TABLE t_sparse_int; From c5ee42d13601a93d723334e4a34aa2442d616c94 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 02:15:48 +0000 Subject: [PATCH 76/81] Backport #125495 to 26.8: Fix row-policy bypass via client-supplied scalar overriding a policy's scalar subquery --- .../evaluateScalarSubqueryIfNeeded.cpp | 2 +- src/Interpreters/Context.cpp | 39 +++++++- src/Interpreters/Context.h | 23 ++++- .../ExecuteScalarSubqueriesVisitor.cpp | 2 +- src/Server/TCPHandler.cpp | 6 +- ...icy_scalar_subquery_cache_bypass.reference | 12 +++ ...row_policy_scalar_subquery_cache_bypass.sh | 97 +++++++++++++++++++ 7 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.reference create mode 100755 tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.sh diff --git a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp index 93d4dfb653cb7..c351e0479e991 100644 --- a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp +++ b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp @@ -143,7 +143,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden scalar_block = scalars_cache.at(node_with_hash); } - else if (context->hasQueryContext() && can_use_global_scalars && context->getQueryContext()->hasScalar(str_hash)) + else if (context->hasQueryContext() && can_use_global_scalars && context->getQueryContext()->hasComputedScalar(str_hash)) { scalar_block = context->getQueryContext()->getScalar(str_hash); scalar_subquery_to_scalar_value_global.emplace(node_with_hash, scalar_block); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 8626af9c9793d..01696d5962828 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1358,6 +1358,15 @@ void ContextSharedMutex::lockSharedImpl() ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds()); } +/// An entry of `ContextData::scalars`. `from_client` is true when the block arrived from the client +/// over the wire (`Context::addClientScalar`) rather than being computed by the server, so that +/// `Context::hasComputedScalar` can keep such a block out of the analyzer's scalar-subquery cache. +struct ScalarEntry +{ + Block block; + bool from_client = false; +}; + ContextData::ContextData() { settings = std::make_unique(); @@ -2664,7 +2673,10 @@ std::pair Context::setConcurrentThreadsSoftLimit(UInt64 num, UIn Scalars Context::getScalars() const { std::lock_guard lock(mutex); - return scalars; + Scalars res; + for (const auto & [name, entry] : scalars) + res.emplace(name, entry.block); + return res; } @@ -2679,7 +2691,7 @@ Block Context::getScalar(const String & name) const // often, so 'bad arguments' for now. throw Exception(ErrorCodes::BAD_ARGUMENTS, "Scalar {} doesn't exist (internal bug)", backQuoteIfNeed(name)); } - return it->second; + return it->second.block; } std::optional Context::tryGetSpecialScalar(const String & name) const @@ -2821,7 +2833,18 @@ void Context::addScalar(const String & name, const Block & block) throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); std::lock_guard lock(mutex); - scalars[name] = block; + /// A server-computed value overrides any client-supplied one of the same name and is trusted. + scalars[name] = ScalarEntry{block, /*from_client=*/ false}; +} + + +void Context::addClientScalar(const String & name, const Block & block) +{ + if (isGlobalContext()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); + + std::lock_guard lock(mutex); + scalars[name] = ScalarEntry{block, /*from_client=*/ true}; } @@ -2844,6 +2867,16 @@ bool Context::hasScalar(const String & name) const return scalars.contains(name); } +bool Context::hasComputedScalar(const String & name) const +{ + if (isGlobalContext()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot have scalars"); + + std::lock_guard lock(mutex); + auto it = scalars.find(name); + return it != scalars.end() && !it->second.from_client; +} + void Context::addQueryAccessInfo( const StorageID & table_id, const Names & column_names) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index cef2e866918a9..916929569a4b1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -174,6 +174,10 @@ class DDLWorker; class ITableFunction; using TableFunctionPtr = std::shared_ptr; class Block; +/// An entry of the query scalar map: the scalar block plus whether it came from the client. +/// Defined in Context.cpp, where `Block` is complete; here `std::map` only needs the forward +/// declaration, the same way `Scalars` is a `std::map` over the forward-declared `Block`. +struct ScalarEntry; class ActionLocksManager; using ActionLocksManagerPtr = std::shared_ptr; class ShellCommand; @@ -418,8 +422,13 @@ class ContextData TemporaryTablesMapping external_tables_mapping; mutable std::shared_ptr hypothetical_object_store; - /// Query scalars - Scalars scalars; + /// Query scalars. Each entry carries whether it was supplied by the client over the wire (the + /// `Scalar` packet, i.e. the `clickhouse-client --external ... --scalar` feature or an initiator + /// shipping scalars to a shard) rather than computed by the server itself. The origin travels with + /// the block so that the analyzer's scalar-subquery cache (which keys on the query-tree hash) never + /// mistakes a client-supplied block for an already-computed subquery result; without this a client + /// could forge the result of a scalar subquery used in a row policy and bypass the policy. + std::map scalars; /// Used to store constant values which are different on each instance during distributed plan, such as _shard_num. Scalars special_scalars; @@ -1058,6 +1067,16 @@ class Context: public ContextData, public std::enable_shared_from_this void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; + /// Add a scalar received from the client over the wire (the `Scalar` packet). Unlike `addScalar`, + /// the entry is marked as client-supplied so that `hasComputedScalar` will not report it as a + /// server-computed subquery result. Explicit readers (`__getScalar`, `_shard_num`, forwarding to + /// shards) still see it through `getScalar` / `hasScalar` / `getScalars`. + void addClientScalar(const String & name, const Block & block); + /// Like `hasScalar`, but returns true only for scalars the server computed itself, never for + /// client-supplied ones. The analyzer's scalar-subquery cache must use this so that a forged + /// client scalar cannot stand in for the result of a (e.g. row-policy) scalar subquery. + bool hasComputedScalar(const String & name) const; + std::optional tryGetSpecialScalar(const String & name) const; void addSpecialScalar(const String & name, const Block & block); diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 6d202197f3795..207c7feb034a5 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -158,7 +158,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr } else { - if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str)) + if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasComputedScalar(scalar_query_hash_str)) { if (!data.getContext()->getViewSource()) { diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 1e3febebb81e6..64da22d89941c 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2891,8 +2891,10 @@ bool TCPHandler::processData(QueryState & state, bool scalar) if (scalar) { - /// Scalar value - state.query_context->addScalar(temporary_id.table_name, block); + /// Scalar value received from the client over the wire. It must not be trusted as a + /// server-computed scalar-subquery result (see Context::addClientScalar): otherwise a client + /// could name it with a row policy's scalar-subquery hash and forge the policy's result. + state.query_context->addClientScalar(temporary_id.table_name, block); } else if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) { diff --git a/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.reference b/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.reference new file mode 100644 index 0000000000000..d9ab9118613ac --- /dev/null +++ b/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.reference @@ -0,0 +1,12 @@ +-- baseline: the policy confines the user to its single tenant (1) +1 +-- with a forged client scalar named as the policy subquery hash, the policy still holds (1) +1 +-- and the visible tenant is unchanged (2) +2 +-- a client scalar is still usable through an explicit __getScalar reference (42) +42 +-- a scalar subquery shipped to the shards still resolves there (8 = 4 tenants x 2 shards) +8 +-- and the shards consumed it through an explicit __getScalar reference (1) +1 diff --git a/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.sh b/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.sh new file mode 100755 index 0000000000000..acba9fbced8f4 --- /dev/null +++ b/tests/queries/0_stateless/05292_row_policy_scalar_subquery_cache_bypass.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# A row policy whose USING expression contains a scalar subquery (here `has((SELECT [2]), tenant_id)`) +# is resolved through the analyzer's scalar-subquery cache, which lives in the same `Context::scalars` +# map that a client can write to with a `Scalar` packet (the `clickhouse-client --external ... --scalar` +# feature). A restricted user could therefore name a client scalar with the policy subquery's query-tree +# hash and forge its result, replacing `(SELECT [2])` with an attacker-chosen array and reading every +# tenant's rows. This test pins that the forged client scalar is ignored by the policy. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# User and policy names are server-global, so suffix them with the (unique) test database. +user="u_scalar_rlp_${CLICKHOUSE_DATABASE}" +policy="rp_scalar_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id UInt64, tenant_id UInt8, secret String) ENGINE = MergeTree ORDER BY id; +INSERT INTO docs SELECT number, number, 'secret-' || toString(number) FROM numbers(1, 8); + +CREATE USER ${user} NOT IDENTIFIED; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.docs TO ${user}; +-- The policy confines ${user} to tenant 2 only, via a scalar subquery. +CREATE ROW POLICY ${policy} ON ${CLICKHOUSE_DATABASE}.docs FOR SELECT USING has((SELECT [2]), tenant_id) TO ${user}; +-- Once a table has any policy, \`throw_on_unmatched_row_policies\` (on in the test config) rejects +-- reads by users no policy matches, so the default user needs a permissive one of its own. +CREATE ROW POLICY ${policy}_all ON ${CLICKHOUSE_DATABASE}.docs FOR SELECT USING 1 TO ALL EXCEPT ${user}; +" + +# Derive the query-tree hash the policy subquery `(SELECT [2])` is cached under. ClickHouse prints it +# itself as the argument of the internal `__getScalar` function (array/tuple scalars are rewritten to +# it), so no access to `system.row_policies` is needed - exactly the disclosure a real attacker uses. +# `enable_scalar_subquery_optimization` is pinned on: with it off the array subquery folds to a literal +# instead of `__getScalar('')`, so the hash would not be derivable and the repro would not run. +hash=$(${CLICKHOUSE_CLIENT} --query "EXPLAIN QUERY TREE SELECT has((SELECT [2]), materialize(1)) SETTINGS enable_scalar_subquery_optimization = 1" \ + | grep -A3 'function_name: __getScalar' | grep -oE "[0-9]+_[0-9]+" | head -1) + +if [ -z "${hash}" ]; then echo "FAILED: could not derive scalar hash"; fi + +# The scalar the attacker injects: an array naming every tenant, so a successful bypass would return all 8 rows. +scalar_file="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_scalar.tsv" +echo '[0,1,2,3,4,5,6,7,8]' > "${scalar_file}" + +echo "-- baseline: the policy confines the user to its single tenant (1)" +${CLICKHOUSE_CLIENT} --user "${user}" --query "SELECT count() FROM ${CLICKHOUSE_DATABASE}.docs SETTINGS enable_scalar_subquery_optimization = 1" + +echo "-- with a forged client scalar named as the policy subquery hash, the policy still holds (1)" +${CLICKHOUSE_CLIENT} --user "${user}" \ + --external --name="${hash}" --scalar --file="${scalar_file}" --structure="x Array(UInt8)" --format=TSV \ + --query "SELECT count() FROM ${CLICKHOUSE_DATABASE}.docs SETTINGS use_query_cache = 0, enable_scalar_subquery_optimization = 1" + +echo "-- and the visible tenant is unchanged (2)" +${CLICKHOUSE_CLIENT} --user "${user}" \ + --external --name="${hash}" --scalar --file="${scalar_file}" --structure="x Array(UInt8)" --format=TSV \ + --query "SELECT DISTINCT tenant_id FROM ${CLICKHOUSE_DATABASE}.docs SETTINGS use_query_cache = 0, enable_scalar_subquery_optimization = 1" + +echo "-- a client scalar is still usable through an explicit __getScalar reference (42)" +echo '42' > "${scalar_file}" +${CLICKHOUSE_CLIENT} --user "${user}" \ + --external --name="my_scalar" --scalar --file="${scalar_file}" --structure="x UInt64" --format=TSV \ + --query "SELECT __getScalar('my_scalar')" + +# Marking wire scalars as client-supplied must not disturb distributed queries: the initiator computes +# an array-returning scalar subquery once and ships the result to the shards in the same `Scalar` packet, +# where it arrives through `addClientScalar`. The shard consumes it through the explicit `__getScalar` +# reader, which still reads client-supplied entries, rather than through the narrowed subquery cache. +log_comment="scalar_rlp_dist_${CLICKHOUSE_DATABASE}" + +echo "-- a scalar subquery shipped to the shards still resolves there (8 = 4 tenants x 2 shards)" +${CLICKHOUSE_CLIENT} --log_comment "${log_comment}" --query " +SELECT count() FROM cluster('test_cluster_two_shards', '${CLICKHOUSE_DATABASE}', docs) +WHERE has((SELECT [2, 3, 4, 5]), tenant_id) +SETTINGS prefer_localhost_replica = 0, enable_scalar_subquery_optimization = 1" + +echo "-- and the shards consumed it through an explicit __getScalar reference (1)" +# The secondary queries log `current_database = 'default'` rather than the test database, so they are +# selected by the id of the initial query, which does carry it. +${CLICKHOUSE_CLIENT} --query " +SYSTEM FLUSH LOGS query_log; +SELECT count() > 0 FROM system.query_log +WHERE event_date >= yesterday() AND type = 'QueryFinish' AND is_initial_query = 0 + AND query LIKE '%__getScalar%' + AND initial_query_id IN ( + SELECT query_id FROM system.query_log + WHERE event_date >= yesterday() AND type = 'QueryFinish' AND is_initial_query = 1 + AND current_database = currentDatabase() AND log_comment = '${log_comment}')" + +rm -f "${scalar_file}" +${CLICKHOUSE_CLIENT} --query " +DROP ROW POLICY ${policy} ON ${CLICKHOUSE_DATABASE}.docs; +DROP ROW POLICY ${policy}_all ON ${CLICKHOUSE_DATABASE}.docs; +DROP TABLE docs; +" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" From 59b09df126085d976b35b9ace046ca2c42e31a7b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 07:35:31 +0000 Subject: [PATCH 77/81] Backport #122688 to 26.8: Substitute query parameters in row policy filters and reject unsubstituted `Identifier` parameters --- src/Analyzer/QueryTreeBuilder.cpp | 26 +++++++++- .../ReplaceQueryParameterVisitor.cpp | 20 ++++++++ src/Parsers/Access/ASTCreateRowPolicyQuery.h | 4 +- src/Parsers/QueryParameterVisitor.cpp | 33 ++++++++++-- src/Parsers/QueryParameterVisitor.h | 4 +- ...n_row_policy_and_filter_settings.reference | 4 ++ ...ters_in_row_policy_and_filter_settings.sql | 51 +++++++++++++++++++ ...query_parameters_in_http_handler.reference | 2 + ...policy_query_parameters_in_http_handler.sh | 31 +++++++++++ 9 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.reference create mode 100644 tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.sql create mode 100644 tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.reference create mode 100755 tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.sh diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 7c0cd9ca34441..0b5e861df1473 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -88,6 +88,16 @@ namespace ErrorCodes namespace { +/// An `Identifier` query parameter that was never substituted: an empty name part, with the parameter as a child. +void throwIfQueryParameter(const ASTIdentifier & identifier) +{ + if (!identifier.isParam()) + return; + + const auto & parameter = static_cast(identifier).children.front()->as(); + throw Exception(ErrorCodes::UNKNOWN_QUERY_PARAMETER, "Query parameter {} was not set", backQuote(parameter.name)); +} + class QueryTreeBuilder { public: @@ -613,11 +623,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co if (const auto * ast_identifier = expression->as()) { + throwIfQueryParameter(*ast_identifier); auto identifier = Identifier(ast_identifier->name_parts); result = std::make_shared(std::move(identifier)); } else if (const auto * table_identifier = expression->as()) { + throwIfQueryParameter(*table_identifier); auto identifier = Identifier(table_identifier->name_parts); result = std::make_shared(std::move(identifier)); } @@ -629,6 +641,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co else if (const auto * qualified_asterisk = expression->as()) { auto & qualified_identifier = qualified_asterisk->qualifier->as(); + throwIfQueryParameter(qualified_identifier); auto column_transformers = buildColumnTransformers(qualified_asterisk->transformers, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), std::move(column_transformers)); } @@ -661,6 +674,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co "Lambda {} argument is not identifier", function->formatForErrorMessage()); + throwIfQueryParameter(*lambda_argument_identifier); + if (lambda_argument_identifier->name_parts.size() > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Lambda {} argument identifier must contain single part. Actual {}", @@ -764,6 +779,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co for (auto & column_list_child : columns_list_matcher->column_list->children) { auto & column_list_identifier = column_list_child->as(); + throwIfQueryParameter(column_list_identifier); column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts}); } @@ -773,12 +789,14 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co else if (const auto * qualified_columns_regexp_matcher = expression->as()) { auto & qualified_identifier = qualified_columns_regexp_matcher->qualifier->as(); + throwIfQueryParameter(qualified_identifier); auto column_transformers = buildColumnTransformers(qualified_columns_regexp_matcher->transformers, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getPattern(), std::move(column_transformers)); } else if (const auto * qualified_columns_list_matcher = expression->as()) { auto & qualified_identifier = qualified_columns_list_matcher->qualifier->as(); + throwIfQueryParameter(qualified_identifier); Identifiers column_list_identifiers; column_list_identifiers.reserve(qualified_columns_list_matcher->column_list->children.size()); @@ -786,6 +804,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co for (auto & column_list_child : qualified_columns_list_matcher->column_list->children) { auto & column_list_identifier = column_list_child->as(); + throwIfQueryParameter(column_list_identifier); column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts}); } @@ -954,6 +973,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(bool is_subquery, const ASTSele if (table_expression.database_and_table_name) { auto & table_identifier_typed = table_expression.database_and_table_name->as(); + throwIfQueryParameter(table_identifier_typed); auto storage_identifier = Identifier(table_identifier_typed.name_parts); QueryTreeNodePtr table_identifier_node; @@ -1225,7 +1245,11 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & except_column_names.reserve(except_transformer->children.size()); for (auto & except_transformer_child : except_transformer->children) - except_column_names.push_back(except_transformer_child->as().full_name); + { + const auto & except_identifier = except_transformer_child->as(); + throwIfQueryParameter(except_identifier); + except_column_names.push_back(except_identifier.full_name); + } column_transformers.emplace_back(std::make_shared(std::move(except_column_names), except_transformer->is_strict)); } diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index ecaed9b0aea0f..d444d8a7b1150 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -104,6 +106,24 @@ void ReplaceQueryParameterVisitor::visit(ASTPtr & ast) else visitChildren(ast); } + else if (auto * create_row_policy_query = dynamic_cast(ast.get())) + { + /// The filters are not children, see ASTCreateRowPolicyQuery::filters. + for (auto & [_, filter] : create_row_policy_query->filters) + { + if (filter) + visit(filter); + } + visitChildren(ast); + } + else if (auto * create_masking_policy_query = dynamic_cast(ast.get())) + { + if (create_masking_policy_query->update_assignments) + visit(create_masking_policy_query->update_assignments); + if (create_masking_policy_query->where_condition) + visit(create_masking_policy_query->where_condition); + visitChildren(ast); + } else visitChildren(ast); } diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.h b/src/Parsers/Access/ASTCreateRowPolicyQuery.h index 7a13206e0c0c9..9407173bc36bf 100644 --- a/src/Parsers/Access/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.h @@ -41,7 +41,9 @@ class ASTCreateRowPolicyQuery : public IAST, public ASTQueryWithOnCluster String new_short_name; std::optional is_restrictive; - std::vector> filters; /// `nullptr` means set to NONE. + /// `nullptr` means set to NONE. The filters are not in `children`: a filter is evaluated later, in the context of + /// the query reading the table, so the generic visitors rewriting this query must not see it. + std::vector> filters; boost::intrusive_ptr roles; diff --git a/src/Parsers/QueryParameterVisitor.cpp b/src/Parsers/QueryParameterVisitor.cpp index 18226daf28701..7364993d858ce 100644 --- a/src/Parsers/QueryParameterVisitor.cpp +++ b/src/Parsers/QueryParameterVisitor.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -14,8 +16,9 @@ namespace DB class QueryParameterVisitor { public: - explicit QueryParameterVisitor(NameToNameMap & parameters) + explicit QueryParameterVisitor(NameToNameMap & parameters, bool include_deferred_expressions_ = true) : query_parameters(parameters) + , include_deferred_expressions(include_deferred_expressions_) { } @@ -27,6 +30,9 @@ class QueryParameterVisitor visitSetQuery(*set_query); else { + if (include_deferred_expressions) + visitDeferredExpressions(*ast); + for (const auto & child : ast->children) visit(child); } @@ -34,6 +40,27 @@ class QueryParameterVisitor private: NameToNameMap & query_parameters; + const bool include_deferred_expressions; + + /// Row policy filters and masking policy expressions are not children, see ASTCreateRowPolicyQuery::filters. + void visitDeferredExpressions(const IAST & ast) + { + if (const auto * create_row_policy_query = ast.as()) + { + for (const auto & [_, filter] : create_row_policy_query->filters) + { + if (filter) + visit(filter); + } + } + else if (const auto * create_masking_policy_query = ast.as()) + { + if (create_masking_policy_query->update_assignments) + visit(create_masking_policy_query->update_assignments); + if (create_masking_policy_query->where_condition) + visit(create_masking_policy_query->where_condition); + } + } void visitQueryParameter(const ASTQueryParameter & query_parameter) { @@ -75,10 +102,10 @@ NameSet analyzeReceiveQueryParams(const std::string & query) return query_param_names; } -NameSet analyzeReceiveQueryParams(const ASTPtr & ast) +NameSet analyzeReceiveQueryParams(const ASTPtr & ast, bool include_deferred_expressions) { NameToNameMap query_params; - QueryParameterVisitor(query_params).visit(ast); + QueryParameterVisitor(query_params, include_deferred_expressions).visit(ast); NameSet query_param_names; for (const auto & query_param : query_params) query_param_names.insert(query_param.first); diff --git a/src/Parsers/QueryParameterVisitor.h b/src/Parsers/QueryParameterVisitor.h index 40b2fa6978fe9..6dd4714b26b16 100644 --- a/src/Parsers/QueryParameterVisitor.h +++ b/src/Parsers/QueryParameterVisitor.h @@ -11,7 +11,9 @@ namespace DB /// Find parameters in a query and collect them into set. NameSet analyzeReceiveQueryParams(const std::string & query); -NameSet analyzeReceiveQueryParams(const ASTPtr & ast); +/// Deferred expressions (row policy filters, masking policy UPDATE / WHERE) are evaluated later, +/// in the context of the query reading the table; a definition check must not see them. +NameSet analyzeReceiveQueryParams(const ASTPtr & ast, bool include_deferred_expressions = true); NameToNameMap analyzeReceiveQueryParamsWithType(const ASTPtr & ast); diff --git a/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.reference b/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.reference new file mode 100644 index 0000000000000..e40ef229fd0a8 --- /dev/null +++ b/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.reference @@ -0,0 +1,4 @@ +identifier 2 +stored without placeholder 1 +alter 2 +whole filter 2 diff --git a/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.sql b/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.sql new file mode 100644 index 0000000000000..533949cc89ef4 --- /dev/null +++ b/tests/queries/0_stateless/05291_query_parameters_in_row_policy_and_filter_settings.sql @@ -0,0 +1,51 @@ +-- Query parameters in CREATE / ALTER ROW POLICY filters are substituted when the policy is stored. +-- The same for MASKING POLICY expressions; masking policies are Cloud-only, so only an unset parameter is checked here. +-- An `Identifier` query parameter in a filter setting is never substituted, so it is rejected. + +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE TABLE t (k UInt64) ENGINE = Memory; +INSERT INTO t VALUES (1), (2); +CREATE TABLE keys (k UInt64, a Array(UInt64)) ENGINE = Memory; +INSERT INTO keys VALUES (1, [1]); +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.keys (k UInt64) ENGINE = Memory; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.keys VALUES (2); + +CREATE ROW POLICY p ON t USING k IN (SELECT k FROM {CLICKHOUSE_DATABASE_1:Identifier}.keys) TO ALL; +SELECT 'identifier', k FROM t ORDER BY k; +SELECT 'stored without placeholder', position(select_filter, '{') = 0 FROM system.row_policies WHERE database = currentDatabase() AND table = 't' AND short_name = 'p'; + +SET param_v = '1'; +ALTER ROW POLICY p ON t USING k > {v:UInt64}; +SELECT 'alter', k FROM t ORDER BY k; + +SET param_cond = '1'; +ALTER ROW POLICY p ON t USING {cond:UInt8}; +SELECT 'whole filter', count() FROM t; + +ALTER ROW POLICY p ON t USING k > {unset_param:UInt64}; -- { serverError UNKNOWN_QUERY_PARAMETER } +DROP ROW POLICY p ON t; + +CREATE ROW POLICY p2 ON t USING k IN (SELECT k FROM keys LEFT ARRAY JOIN 'abc', {unset_param:Identifier}) TO ALL; -- { serverError UNKNOWN_QUERY_PARAMETER } +SET param_col = 'a'; +CREATE ROW POLICY p2 ON t USING k IN (SELECT k FROM keys LEFT ARRAY JOIN 'abc', {col:Identifier}) TO ALL; +SELECT k FROM t; -- { serverError TYPE_MISMATCH } +DROP ROW POLICY p2 ON t; + +CREATE MASKING POLICY mp ON t UPDATE k = {unset_param:UInt64} TO ALL; -- { serverError UNKNOWN_QUERY_PARAMETER } +ALTER MASKING POLICY mp ON t UPDATE k = 0 WHERE k > {unset_param:UInt64}; -- { serverError UNKNOWN_QUERY_PARAMETER } + +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT k FROM keys LEFT ARRAY JOIN \'abc\', {x:Identifier})'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_result_filter = 'k IN (SELECT k FROM keys LEFT ARRAY JOIN \'abc\', {x:Identifier})'; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT COLUMNS({x:Identifier}) FROM keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT k FROM {x:Identifier}.keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'arrayExists({x:Identifier} -> 1, [k])'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT * EXCEPT ({x:Identifier}) FROM keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT {x:Identifier}.* FROM keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT {x:Identifier}.COLUMNS(k) FROM keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } +SELECT k FROM t SETTINGS additional_table_filters = {'t': 'k IN (SELECT {x:Identifier}.COLUMNS(\'k\') FROM keys)'}; -- { serverError UNKNOWN_QUERY_PARAMETER } + +DROP ROW POLICY IF EXISTS p ON t; +DROP ROW POLICY IF EXISTS p2 ON t; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.reference b/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.reference new file mode 100644 index 0000000000000..4968932a1f5d4 --- /dev/null +++ b/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.reference @@ -0,0 +1,2 @@ +2 +BAD_QUERY_PARAMETER diff --git a/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.sh b/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.sh new file mode 100755 index 0000000000000..b1839c377fdc8 --- /dev/null +++ b/tests/queries/0_stateless/05292_row_policy_query_parameters_in_http_handler.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An HTTP handler whose query is CREATE ROW POLICY binds the query parameters of the policy filter from the request. +# The same for CREATE MASKING POLICY; it is Cloud-only, so a value that does not parse shows that the parameter was bound. + +BASE="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}" +DB="${CLICKHOUSE_DATABASE}" +H="hrp_${DB}" +P="/hrp_${DB}" +HM="hmp_${DB}" +PM="/hmp_${DB}" + +cleanup() { + $CLICKHOUSE_CLIENT -q "DROP HANDLER IF EXISTS \`$H\`; DROP HANDLER IF EXISTS \`$HM\`; DROP ROW POLICY IF EXISTS p ON ${DB}.t" +} +trap cleanup EXIT +cleanup + +$CLICKHOUSE_CLIENT -q " +CREATE TABLE ${DB}.t (k UInt64) ENGINE = Memory; +INSERT INTO ${DB}.t VALUES (1), (2); +CREATE HANDLER \`$H\` URL '${P}' METHODS (POST) AS CREATE ROW POLICY p ON ${DB}.t USING k > {v:UInt64} TO ALL; +CREATE HANDLER \`$HM\` URL '${PM}' METHODS (POST) AS CREATE MASKING POLICY mp ON ${DB}.t UPDATE k = {v:UInt64} TO ALL" + +${CLICKHOUSE_CURL} -sS -X POST "${BASE}${P}?param_v=1" +$CLICKHOUSE_CLIENT -q "SELECT k FROM ${DB}.t ORDER BY k" +${CLICKHOUSE_CURL} -sS -X POST "${BASE}${PM}?param_v=abc" | grep -o -E 'UNKNOWN_QUERY_PARAMETER|BAD_QUERY_PARAMETER|SUPPORT_IS_DISABLED' From 73b22d4f65244af7d07d2c29d7f28186966eef9d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 08:30:17 +0000 Subject: [PATCH 78/81] Backport #123188 to 26.8: add a setting to allow disabling interserver mode over tcp_with_proxy interface --- programs/server/config.xml | 3 + programs/server/config.yaml.example | 4 + src/Core/ServerSettings.cpp | 3 + src/Server/TCPHandler.cpp | 21 +++ .../__init__.py | 0 .../configs/config.d/config.xml | 15 ++ .../config.d/disallow_interserver_mode.xml | 4 + .../configs/config.d/session_log.xml | 7 + .../test.py | 158 ++++++++++++++++++ 9 files changed, 215 insertions(+) create mode 100644 tests/integration/test_tcp_with_proxy_interserver_mode/__init__.py create mode 100644 tests/integration/test_tcp_with_proxy_interserver_mode/configs/config.d/config.xml create mode 100644 tests/integration/test_tcp_with_proxy_interserver_mode/configs/config.d/disallow_interserver_mode.xml create mode 100644 tests/integration/test_tcp_with_proxy_interserver_mode/configs/config.d/session_log.xml create mode 100644 tests/integration/test_tcp_with_proxy_interserver_mode/test.py diff --git a/programs/server/config.xml b/programs/server/config.xml index 40eb34dd922c6..7a219ee039beb 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -227,6 +227,9 @@ ClickHouse will extract information about proxy-forwarded client address from the header. --> + +