Skip to content

[fix](variant) Preserve empty key during variant compaction #64641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldenmoon merged 1 commit into from
Jul 2, 2026
Merged

[fix](variant) Preserve empty key during variant compaction #64641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions be/src/exec/common/variant_util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,44 @@

namespace doris::variant_util {

namespace {

PathInData make_full_subcolumn_path(const TabletColumnPtr& parent_column, std::string_view path) {
if (!path.empty()) {
return PathInData(parent_column->name_lower_case() + "." + std::string(path));
}

// Keep the empty JSON key as a real path part. The variant root is `parts.empty()`;
// an empty key is `parts.size() == 1 && parts[0].key.empty()` after popping root.
PathInDataBuilder builder;
return builder.append(parent_column->name_lower_case(), false).append("", false).build();
}

void append_empty_key_subcolumn_from_stats(TabletSchema::PathsSetInfo& paths_set_info,
const TabletColumnPtr& parent_column,
TabletSchemaSPtr& output_schema) {
if (!paths_set_info.sub_path_set.contains("") || paths_set_info.sparse_path_set.contains("") ||
paths_set_info.subcolumn_indexes.contains("")) {
return;
}

auto column_name = parent_column->name_lower_case() + ".";
auto column_path = make_full_subcolumn_path(parent_column, "");

TabletColumn subcolumn;
subcolumn.set_name(column_name);
subcolumn.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
subcolumn.set_parent_unique_id(parent_column->unique_id());
subcolumn.set_path_info(column_path);
subcolumn.set_aggregation_method(parent_column->aggregation());
subcolumn.set_variant_max_subcolumns_count(parent_column->variant_max_subcolumns_count());
subcolumn.set_variant_enable_doc_mode(parent_column->variant_enable_doc_mode());
subcolumn.set_is_nullable(true);
output_schema->append_column(subcolumn);
}

} // namespace

inline void append_escaped_regex_char(std::string* regex_output, char ch) {
switch (ch) {
case '.':
Expand Down Expand Up @@ -1194,7 +1232,8 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
// append subcolumns
for (const auto& subpath : sorted_subpaths) {
auto column_name = parent_column->name_lower_case() + "." + subpath.to_string();
auto column_path = PathInData(column_name);
auto column_path = make_full_subcolumn_path(parent_column,
std::string_view(subpath.data, subpath.size));

const auto& find_data_types = path_to_data_types.find(PathInData(subpath));

Expand Down Expand Up @@ -1261,7 +1300,7 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
DataTypePtr data_type;
get_least_supertype_jsonb(data_types, &data_type);
auto column_name = parent_column->name_lower_case() + "." + path.get_path();
auto column_path = PathInData(column_name);
auto column_path = make_full_subcolumn_path(parent_column, path.get_path());
TabletColumn sub_column =
get_column_by_type(data_type, column_name,
ExtraInfo {.unique_id = -1,
Expand All @@ -1276,6 +1315,10 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
VLOG_DEBUG << "append sub column " << path.get_path() << " data type "
<< data_type->get_name();
}
// The data-type map can contain the variant root as PathInData(), while an empty JSON key
// only appears in path stats as "". If stats selected it as a materialized path, append it
// with an explicit empty path part so it does not collide with root.
append_empty_key_subcolumn_from_stats(paths_set_info, parent_column, output_schema);
}

// Build the temporary schema for compaction.
Expand Down
121 changes: 113 additions & 8 deletions be/test/exec/common/schema_util_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,46 @@ TEST_F(SchemaUtilTest, get_subpaths_equal_to_max) {
uid_to_paths_set_info[1].sub_path_set.end());
}

TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) {
variant_util::PathToNoneNullValues path_stats = {
{"", 1000}, {"path1", 900}, {"path2", 800}, {"path3", 700}};

TabletSchema::PathsSetInfo limited_paths;
variant_util::VariantCompactionUtil::get_subpaths(2, path_stats, limited_paths);
EXPECT_TRUE(limited_paths.sub_path_set.contains(""));
EXPECT_FALSE(limited_paths.sparse_path_set.contains(""));
EXPECT_TRUE(limited_paths.sub_path_set.contains("path1"));
EXPECT_TRUE(limited_paths.sparse_path_set.contains("path2"));
EXPECT_TRUE(limited_paths.sparse_path_set.contains("path3"));

TabletSchema::PathsSetInfo exact_limit_paths;
variant_util::VariantCompactionUtil::get_subpaths(4, path_stats, exact_limit_paths);
EXPECT_TRUE(exact_limit_paths.sub_path_set.contains(""));
EXPECT_FALSE(exact_limit_paths.sparse_path_set.contains(""));
EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path1"));
EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path2"));
EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path3"));

TabletSchema::PathsSetInfo unlimited_paths;
variant_util::VariantCompactionUtil::get_subpaths(0, path_stats, unlimited_paths);
EXPECT_TRUE(unlimited_paths.sub_path_set.contains(""));
EXPECT_TRUE(unlimited_paths.sparse_path_set.empty());
EXPECT_FALSE(unlimited_paths.sparse_path_set.contains(""));
EXPECT_TRUE(unlimited_paths.sub_path_set.contains("path1"));
EXPECT_TRUE(unlimited_paths.sub_path_set.contains("path2"));
EXPECT_TRUE(unlimited_paths.sub_path_set.contains("path3"));

variant_util::PathToNoneNullValues low_rank_empty_key_stats = {
{"path1", 1000}, {"path2", 900}, {"", 100}};
TabletSchema::PathsSetInfo low_rank_empty_key_paths;
variant_util::VariantCompactionUtil::get_subpaths(2, low_rank_empty_key_stats,
low_rank_empty_key_paths);
EXPECT_FALSE(low_rank_empty_key_paths.sub_path_set.contains(""));
EXPECT_TRUE(low_rank_empty_key_paths.sparse_path_set.contains(""));
EXPECT_TRUE(low_rank_empty_key_paths.sub_path_set.contains("path1"));
EXPECT_TRUE(low_rank_empty_key_paths.sub_path_set.contains("path2"));
}

TEST_F(SchemaUtilTest, get_subpaths_multiple_variants) {
TabletSchema schema;
TabletColumn variant1;
Expand Down Expand Up @@ -1522,6 +1562,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) {

TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) {
TabletColumn variant;
variant.set_name("v1");
variant.set_unique_id(30);
variant.set_variant_max_subcolumns_count(3);
variant.set_aggregation_method(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE);
Expand All @@ -1532,6 +1573,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) {
TabletColumnPtr parent_column = std::make_shared<TabletColumn>(variant);

TabletSchema::PathsSetInfo paths_set_info;
paths_set_info.sub_path_set.insert("");
paths_set_info.sub_path_set.insert("a");
paths_set_info.sub_path_set.insert("b");
doris::variant_util::PathToDataTypes path_to_data_types;
Expand All @@ -1540,37 +1582,51 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) {

variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema);
EXPECT_EQ(output_schema->num_columns(), 2);
EXPECT_EQ(output_schema->num_columns(), 3);
bool found_empty_key = false;
for (const auto& column : output_schema->columns()) {
if (column->name() == "v1.") {
found_empty_key = true;
const auto relative_path = column->path_info_ptr()->copy_pop_front();
EXPECT_FALSE(relative_path.empty());
EXPECT_TRUE(relative_path.get_path().empty());
ASSERT_EQ(relative_path.get_parts().size(), 1);
EXPECT_TRUE(relative_path.get_parts()[0].key.empty());
}
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_VARIANT);
}
EXPECT_TRUE(found_empty_key);

output_schema = std::make_shared<TabletSchema>();
path_to_data_types.clear();
path_to_data_types[PathInData("a")] = {std::make_shared<DataTypeInt32>()};
path_to_data_types[PathInData("b")] = {std::make_shared<DataTypeString>()};
variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema);
EXPECT_EQ(output_schema->num_columns(), 2);
EXPECT_EQ(output_schema->num_columns(), 3);
bool found_int = false, found_str = false;
found_empty_key = false;
for (const auto& column : output_schema->columns()) {
if (column->name().ends_with("a")) {
if (column->name() == "v1.") {
found_empty_key = true;
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_VARIANT);
} else if (column->name().ends_with("a")) {
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_INT);
found_int = true;
} else if (column->name().ends_with("b")) {
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_STRING);
found_str = true;
}
}
EXPECT_TRUE(found_int && found_str);
EXPECT_TRUE(found_empty_key && found_int && found_str);

output_schema = std::make_shared<TabletSchema>();
sparse_paths.insert("a");
variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema);
EXPECT_EQ(output_schema->num_columns(), 2);
EXPECT_EQ(output_schema->num_columns(), 3);
for (const auto& column : output_schema->columns()) {
if (column->name().ends_with("a")) {
if (column->name() == "v1." || column->name().ends_with("a")) {
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_VARIANT);
} else if (column->name().ends_with("b")) {
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_STRING);
Expand All @@ -1585,7 +1641,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) {
}
variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema);
EXPECT_EQ(output_schema->num_columns(), 2);
EXPECT_EQ(output_schema->num_columns(), 3);
for (const auto& column : output_schema->columns()) {
EXPECT_EQ(column->type(), FieldType::OLAP_FIELD_TYPE_VARIANT);
}
Expand Down Expand Up @@ -1704,15 +1760,18 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) {
path_to_data_types[PathInData("b")] = {std::make_shared<DataTypeString>()}; // -> STRING
path_to_data_types[PathInData("typed", true)] = {std::make_shared<DataTypeString>()};
path_to_data_types[PathInData("shared")] = {std::make_shared<DataTypeInt32>()};
path_to_data_types[PathInData("")] = {std::make_shared<DataTypeString>()};
path_to_data_types[PathInData()] = {std::make_shared<DataTypeString>()};

TabletSchemaSPtr output_schema = std::make_shared<TabletSchema>();
TabletSchema::PathsSetInfo paths_set_info;

variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
paths_set_info, parent_column, target, path_to_data_types, output_schema);

EXPECT_EQ(output_schema->num_columns(), 3);
EXPECT_EQ(output_schema->num_columns(), 4);
bool found_a = false, found_b = false, found_typed = false, found_shared = false;
int empty_key_column_count = 0;
for (const auto& col : output_schema->columns()) {
if (col->name() == "v1.a") {
found_a = true;
Expand All @@ -1737,9 +1796,19 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) {
EXPECT_EQ(col->type(), FieldType::OLAP_FIELD_TYPE_INT);
EXPECT_EQ(col->parent_unique_id(), 1);
EXPECT_EQ(col->path_info_ptr()->get_path(), "v1.shared");
} else if (col->name() == "v1.") {
++empty_key_column_count;
EXPECT_EQ(col->type(), FieldType::OLAP_FIELD_TYPE_STRING);
EXPECT_EQ(col->parent_unique_id(), 1);
const auto relative_path = col->path_info_ptr()->copy_pop_front();
EXPECT_FALSE(relative_path.empty());
EXPECT_TRUE(relative_path.get_path().empty());
ASSERT_EQ(relative_path.get_parts().size(), 1);
EXPECT_TRUE(relative_path.get_parts()[0].key.empty());
}
}
EXPECT_TRUE(found_a && found_b && found_shared);
EXPECT_EQ(empty_key_column_count, 1);
EXPECT_FALSE(found_typed);

ASSERT_TRUE(paths_set_info.subcolumn_indexes.find("a") !=
Expand All @@ -1751,11 +1820,47 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) {
EXPECT_FALSE(paths_set_info.subcolumn_indexes.contains("typed"));
ASSERT_TRUE(paths_set_info.subcolumn_indexes.contains("shared"));
EXPECT_EQ(paths_set_info.subcolumn_indexes.at("shared").size(), 1);
ASSERT_TRUE(paths_set_info.subcolumn_indexes.contains(""));
EXPECT_EQ(paths_set_info.subcolumn_indexes.at("").size(), 1);
EXPECT_FALSE(paths_set_info.typed_path_set.contains("typed"));
EXPECT_TRUE(paths_set_info.sub_path_set.contains("a"));
EXPECT_TRUE(paths_set_info.sub_path_set.contains("b"));
EXPECT_TRUE(paths_set_info.sub_path_set.contains("shared"));
EXPECT_FALSE(paths_set_info.sub_path_set.contains("typed"));
EXPECT_TRUE(paths_set_info.sub_path_set.contains(""));
EXPECT_FALSE(paths_set_info.sparse_path_set.contains(""));

doris::variant_util::PathToDataTypes root_path_to_data_types;
root_path_to_data_types[PathInData()] = {std::make_shared<DataTypeString>()};
TabletSchemaSPtr root_output_schema = std::make_shared<TabletSchema>();
TabletSchema::PathsSetInfo root_paths_set_info;

variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
root_paths_set_info, parent_column, target, root_path_to_data_types,
root_output_schema);

EXPECT_EQ(root_output_schema->num_columns(), 0);
EXPECT_FALSE(root_paths_set_info.sparse_path_set.contains(""));
EXPECT_FALSE(root_paths_set_info.sub_path_set.contains(""));

TabletSchemaSPtr empty_key_output_schema = std::make_shared<TabletSchema>();
TabletSchema::PathsSetInfo empty_key_paths_set_info;
empty_key_paths_set_info.sub_path_set.insert("");

variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
empty_key_paths_set_info, parent_column, target, root_path_to_data_types,
empty_key_output_schema);

ASSERT_EQ(empty_key_output_schema->num_columns(), 1);
const auto& empty_key_column = empty_key_output_schema->column(0);
EXPECT_EQ(empty_key_column.name(), "v1.");
EXPECT_EQ(empty_key_column.type(), FieldType::OLAP_FIELD_TYPE_VARIANT);
EXPECT_EQ(empty_key_column.parent_unique_id(), 1);
const auto relative_path = empty_key_column.path_info_ptr()->copy_pop_front();
EXPECT_FALSE(relative_path.empty());
EXPECT_TRUE(relative_path.get_path().empty());
ASSERT_EQ(relative_path.get_parts().size(), 1);
EXPECT_TRUE(relative_path.get_parts()[0].key.empty());
}

// Test has_different_structure_in_same_path function indirectly through check_variant_has_no_ambiguous_paths
Expand Down
10 changes: 10 additions & 0 deletions regression-test/data/variant_p0/test_variant_empty_key_sparse_bucket.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !empty_key_no_sparse_values --
1 BEFORE RENAME
2 42
3 AFTER RENAME

-- !empty_key_sparse_values --
7 UPPER CASE
8 16
9 8888888
94 changes: 94 additions & 0 deletions regression-test/suites/variant_p0/test_variant_empty_key_sparse_bucket.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_variant_empty_key_sparse_bucket", "nonConcurrent") {
sql "SET default_variant_enable_doc_mode = false"
sql "SET use_v3_storage_format = false"
sql "SET default_variant_enable_typed_paths_to_sparse = false"
sql "SET default_variant_sparse_hash_shard_count = 3"
sql "SET enable_rewrite_element_at_to_slot = true"

sql "SET default_variant_max_subcolumns_count = 0"
sql "DROP TABLE IF EXISTS test_variant_empty_key_no_sparse"
sql """
CREATE TABLE test_variant_empty_key_no_sparse (
k bigint,
v variant
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 1
properties("replication_num" = "1", "disable_auto_compaction" = "true");
"""

sql """
INSERT INTO test_variant_empty_key_no_sparse VALUES
(1, '{"hot_a": 1, "": "BEFORE RENAME"}'),
(2, '{"hot_a": 2, "": 42}')
"""
sql "ALTER TABLE test_variant_empty_key_no_sparse RENAME COLUMN v Tags"
sql """
INSERT INTO test_variant_empty_key_no_sparse VALUES
(3, '{"hot_a": 3, "": "AFTER RENAME"}')
"""

trigger_and_wait_compaction("test_variant_empty_key_no_sparse", "cumulative")

qt_empty_key_no_sparse_values """
SELECT k, cast(Tags[''] as text)
FROM test_variant_empty_key_no_sparse
WHERE Tags[''] IS NOT NULL
ORDER BY k
"""

sql "SET default_variant_max_subcolumns_count = 3"
sql "DROP TABLE IF EXISTS test_variant_empty_key_sparse_bucket"
sql """
CREATE TABLE test_variant_empty_key_sparse_bucket (
k bigint,
v variant
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 1
properties("replication_num" = "1", "disable_auto_compaction" = "true");
"""

sql """
INSERT INTO test_variant_empty_key_sparse_bucket VALUES
(1, '{"hot_a": 1, "hot_b": 10, "hot_c": 100, "cold_1": "a"}'),
(2, '{"hot_a": 2, "hot_b": 20, "hot_c": 200, "cold_2": "b"}'),
(3, '{"hot_a": 3, "hot_b": 30, "hot_c": 300, "cold_3": "c"}'),
(4, '{"hot_a": 4, "hot_b": 40, "hot_c": 400, "cold_4": "d"}'),
(5, '{"hot_a": 5, "hot_b": 50, "hot_c": 500, "cold_5": "e"}'),
(6, '{"hot_a": 6, "hot_b": 60, "hot_c": 600, "cold_6": "f"}')
"""
sql "ALTER TABLE test_variant_empty_key_sparse_bucket RENAME COLUMN v Tags"
sql """
INSERT INTO test_variant_empty_key_sparse_bucket VALUES
(7, '{"hot_a": 7, "hot_b": 70, "hot_c": 700, "": "UPPER CASE"}'),
(8, '{"hot_a": 8, "hot_b": 80, "hot_c": 800, "": 16}'),
(9, '{"hot_a": 9, "hot_b": 90, "hot_c": 900, "": 8888888}')
"""

trigger_and_wait_compaction("test_variant_empty_key_sparse_bucket", "cumulative")

qt_empty_key_sparse_values """
SELECT k, cast(Tags[''] as text)
FROM test_variant_empty_key_sparse_bucket
WHERE Tags[''] IS NOT NULL
ORDER BY k
"""
}
Loading