From 9d24026e9642c44ebefb495aab6bc682d240f60b Mon Sep 17 00:00:00 2001 From: Chenyang Sun Date: Wed, 3 Jun 2026 11:04:10 +0800 Subject: [PATCH] [chore](compaction) remove single replica compaction (#63771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the single replica compaction (SRC) feature end-to-end across BE, FE and regression tests. Doc: https://github.com/apache/doris-website/pull/3870 ### Why remove it The main reason is **correctness risk in peer selection**. A follower replica had to pick a peer holding a "proper" version (`_find_rowset_to_fetch`) and fetch its compacted result, based on replica info that was only refreshed periodically. Because replicas progress through versions independently and this "leader" selection ran against a stale, time-sensitive view of the cluster, the hoice of which peer to fetch from — and which version — was racy and could select a peer whose state no longer matched, leading to subtle inconsistencies. --- be/src/agent/task_worker_pool.cpp | 12 - be/src/cloud/pb_convert.cpp | 4 - be/src/common/config.cpp | 4 - be/src/common/config.h | 3 - be/src/common/metrics/doris_metrics.cpp | 6 - be/src/common/metrics/doris_metrics.h | 3 - .../service/http/action/compaction_action.cpp | 30 +- .../service/http/action/compaction_action.h | 4 +- .../compaction/single_replica_compaction.cpp | 589 ------------------ .../compaction/single_replica_compaction.h | 70 --- be/src/storage/olap_server.cpp | 200 ------ be/src/storage/storage_engine.cpp | 42 -- be/src/storage/storage_engine.h | 18 - be/src/storage/tablet/tablet.cpp | 55 -- be/src/storage/tablet/tablet.h | 14 - be/src/storage/tablet/tablet_manager.cpp | 34 +- be/src/storage/tablet/tablet_meta.cpp | 5 - be/src/storage/tablet/tablet_schema.cpp | 4 - be/src/storage/tablet/tablet_schema.h | 5 - .../service/http/compaction_action_test.cpp | 13 - .../compaction/single_compaction_test.cpp | 147 ----- be/test/storage/tablet/tablet_mgr_test.cpp | 64 +- .../test_data/header_without_inc_rs.txt | 1 - .../org/apache/doris/alter/RollupJobV2.java | 1 - .../doris/alter/SchemaChangeHandler.java | 23 +- .../apache/doris/alter/SchemaChangeJobV2.java | 1 - .../analysis/ModifyTablePropertiesClause.java | 10 - .../org/apache/doris/backup/RestoreJob.java | 1 - .../java/org/apache/doris/catalog/Env.java | 5 - .../org/apache/doris/catalog/OlapTable.java | 16 - .../apache/doris/catalog/TableProperty.java | 14 - .../doris/common/util/PropertyAnalyzer.java | 22 - .../doris/datasource/InternalCatalog.java | 18 +- .../apache/doris/master/ReportHandler.java | 1 - .../info/ModifyTablePropertiesOp.java | 11 - .../apache/doris/persist/OperationType.java | 2 +- .../apache/doris/task/CreateReplicaTask.java | 5 - .../doris/task/UpdateTabletMetaInfoTask.java | 6 - .../nereids/minidump/MinidumpUtTestData.json | 1 - .../rules/rewrite/PruneNestedColumnTest.java | 3 +- .../org/apache/doris/task/AgentTaskTest.java | 2 +- gensrc/proto/olap_file.proto | 4 +- gensrc/thrift/AgentService.thrift | 4 +- .../ttl/ex06_ttl_restart_consistency.out | 4 +- .../data/cloud_p0/cache/ttl/test_show_ttl.out | 4 +- ...test_single_compaction_fault_injection.out | 13 - ...compaction_with_variant_inverted_index.out | 4 - .../test_single_replica_compaction.out | 10 - ...test_group_commit_interval_ms_property.out | 8 +- ...ngle_replica_compaction_with_format_v2.out | 9 - .../query_p0/show/test_show_create_table.out | 4 +- .../query_p0/system/test_table_properties.out | 8 +- ...st_show_create_table_and_views_nereids.out | 8 +- .../show_p0/test_show_table_and_views.out | 4 +- ...t_single_compaction_fault_injection.groovy | 342 ---------- ...paction_with_variant_inverted_index.groovy | 257 -------- .../test_single_replica_compaction.groovy | 264 -------- .../test_table_level_compaction_policy.groovy | 8 - .../suites/correctness_p0/test_and_or.groovy | 3 +- .../correctness_p0/test_cast_null.groovy | 9 +- ...olocate_join_with_different_tablets.groovy | 4 +- .../test_ctas_mv/test_ctas_mv.groovy | 6 +- .../test_mv_case/test_mv_case.groovy | 3 - .../test_nested_lateral_view.groovy | 3 +- .../correctness_p0/test_set_operation.groovy | 6 +- .../decimalv2/test_decimalv2_overflow2.groovy | 3 +- .../decimalv3/test_decimalv3_string.groovy | 2 +- .../test_large_string_and_nested_type.groovy | 1 - .../test_special_buffer_before_nested.groovy | 1 - .../query/test_nested_type_with_resize.groovy | 3 +- .../test_create_view_nereids.groovy | 3 +- .../stream_load_and_mysql_load.groovy | 1 - .../suites/ddl_p0/test_ctas.groovy | 3 +- .../ddl_p0/test_drop_view_nereids.groovy | 3 +- .../suites/delete_p0/test_delete.groovy | 3 +- ..._with_complex_type_insertinto_doris.groovy | 6 +- .../test_build_index_fault.groovy | 3 +- .../suites/function_p0/test_array_map.groovy | 1 - .../test_iot_auto_detect_fail.groovy | 2 - .../test_array_contains_estimate.groovy | 3 +- .../index_change/test_build_index.groovy | 3 +- ...e_replica_compaction_with_format_v2.groovy | 208 ------- .../test_arrayInvertedIdx_profile.groovy | 3 +- ..._array_contains_with_inverted_index.groovy | 3 +- .../inverted_index_p0/test_array_index.groovy | 6 +- ..._array_with_inverted_index_all_type.groovy | 14 +- .../inverted_index_p0/test_delete.groovy | 3 +- .../test_index_rqg_bug8.groovy | 1 - ..._ip_cidr_search_with_inverted_index.groovy | 3 +- .../stream_load/test_csv_split_line.groovy | 3 +- .../test_csv_with_double_quotes.groovy | 3 +- .../load_p0/stream_load/test_json_load.groovy | 3 +- .../test_load_with_map_nested_array.groovy | 3 +- .../test_load_with_transfer_encoding.groovy | 1 - .../test_stream_load_error_url.groovy | 1 - .../ddl/test_multi_char_line_delimiter.sql | 1 - .../mv_with_force_drop.groovy | 3 +- .../routine_load_mapping.groovy | 1 - .../suites/mv_p0/test_tcu/test_tcu.groovy | 2 +- .../nereids_arith_p0/topn/accept_null.groovy | 1 - .../aggregate/agg_distinct_case_when.groovy | 3 +- .../suites/nereids_p0/eager_agg/load.groovy | 1 - .../nereids_p0/explain/explain_graph.groovy | 2 - .../explain/test_pushdown_explain.groovy | 6 +- .../insert_into_table/partial_update.groovy | 1 - .../runtime_filter/expr-target.groovy | 2 - .../nereids_p0/subquery/test_subquery.groovy | 6 +- .../subquery/test_subquery_in_project.groovy | 3 +- .../suites/nereids_p0/tablesample.groovy | 3 +- .../test_forbid_unknown_col_stats.groovy | 3 +- .../test_repeat_no_stackflow.groovy | 6 - .../update/update_after_create_rollup.groovy | 1 - .../suites/nereids_rules_p0/cse/cse.groovy | 1 - .../cte/test_cte_name_reuse.groovy | 3 +- .../defer_materialize_topn/one_phase.groovy | 1 - .../test_grouping_sets_combination.groovy | 6 +- .../union_all_compensate.groovy | 2 - .../partition_prune/prune_or.groovy | 1 - .../test_multi_range_partition.groovy | 3 +- .../runtime_filter/rf_prune.groovy | 2 - .../transposeJoin/transposeSemiJoinAgg.groovy | 2 - .../nereids_syntax_p0/join_order.groovy | 9 +- .../push_filter_through_ptopn.groovy | 3 +- .../push_filter_through_window.groovy | 3 +- .../prepared_stmt_p0/prepared_stmt.groovy | 1 - .../aggregate/aggregate_not_group_null.groovy | 3 +- .../query_p0/aggregate/array_agg.groovy | 12 +- .../query_p0/aggregate/bitmap_agg.groovy | 6 +- .../suites/query_p0/aggregate/map_agg.groovy | 12 +- .../map_agg_nested_insert_doris.groovy | 6 +- .../query_p0/aggregate/map_agg_v1.groovy | 12 +- .../query_p0/cache/condition_cache.groovy | 6 +- .../suites/query_p0/cache/query_cache.groovy | 2 - .../cast/test_complextype_to_json.groovy | 3 +- .../suites/query_p0/having/having.groovy | 3 +- .../join/rqg/doris8317/doris8317.groovy | 3 +- .../join/test_collect_list_distinct.groovy | 9 +- .../join/test_join_with_cast_array.groovy | 15 +- .../operator/test_sort_operator.groovy | 3 +- .../rqg_prod_20240226.groovy | 2 - .../test_aggregate_all_functions2.groovy | 3 +- ...count_distinct_with_collection_type.groovy | 3 +- .../sql_functions/array_functions/sql/q04.sql | 3 +- .../test_array_map_function.groovy | 3 +- .../table_function/explode.groovy | 7 +- .../explode_array_decimal.groovy | 1 - .../system/test_query_sys_rowsets.groovy | 3 +- .../system/test_query_sys_scan_rowsets.groovy | 3 +- .../system/test_query_sys_tables.groovy | 3 +- .../suites/query_p0/view/test_view2.groovy | 6 +- .../char_type_shrink_before_project.groovy | 1 - regression-test/suites/row_store/load.groovy | 6 +- .../test_alter_table_property.groovy | 17 - .../test_alter_rollup_table.groovy | 1 - .../test_schema_change_delete.groovy | 1 - ...ow_create_table_with_storage_policy.groovy | 3 +- .../suites/statistics/analyze_stats.groovy | 3 +- .../statistics/test_scale_num_nulls.groovy | 1 - .../map_uniq_with_local_tvf.groovy | 3 +- ...t_partial_update_native_insert_stmt.groovy | 1 - .../test_partial_update_strict_mode.groovy | 5 - .../test_partial_update_upsert.groovy | 2 - ...t_partial_update_native_insert_stmt.groovy | 1 - .../test_partial_update_strict_mode.groovy | 5 - .../test_partial_update_upsert.groovy | 2 - .../test_create_table.groovy | 20 - .../suites/variant_p0/doc_mode/load.groovy | 1 - .../variant_p0/doc_mode/predefine/load.groovy | 1 - .../test_predefine_type_index.groovy | 1 - ..._array_contains_with_inverted_index.groovy | 1 - regression-test/suites/variant_p0/load.groovy | 1 - .../suites/variant_p0/mv/multi_slot.groovy | 1 - .../suites/variant_p0/predefine/load.groovy | 1 - .../test_predefine_type_index.groovy | 1 - .../suites/variant_p0/rqg/fix_rqg1.groovy | 1 - .../suites/variant_p0/rqg/rqg6.sql | 1 - .../suites/variant_p0/rqg/rqg7.sql | 4 - ..._array_contains_with_inverted_index.groovy | 1 - .../test_array_inverted_index_profile.groovy | 1 - tools/coffeebench-tools/ddl/create-tables.sql | 3 - 180 files changed, 146 insertions(+), 2913 deletions(-) delete mode 100644 be/src/storage/compaction/single_replica_compaction.cpp delete mode 100644 be/src/storage/compaction/single_replica_compaction.h delete mode 100644 be/test/storage/compaction/single_compaction_test.cpp delete mode 100644 regression-test/data/compaction/test_single_compaction_fault_injection.out delete mode 100644 regression-test/data/compaction/test_single_compaction_with_variant_inverted_index.out delete mode 100644 regression-test/data/compaction/test_single_replica_compaction.out delete mode 100644 regression-test/data/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.out delete mode 100644 regression-test/suites/compaction/test_single_compaction_fault_injection.groovy delete mode 100644 regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy delete mode 100644 regression-test/suites/compaction/test_single_replica_compaction.groovy delete mode 100644 regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 022ce183a1af6d..de19f59b5ac172 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -998,18 +998,6 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest& tablet->set_binlog_config(new_binlog_config); need_to_save = true; } - if (tablet_meta_info.__isset.enable_single_replica_compaction) { - std::shared_lock rlock(tablet->get_header_lock()); - tablet->tablet_meta()->mutable_tablet_schema()->set_enable_single_replica_compaction( - tablet_meta_info.enable_single_replica_compaction); - for (auto& [_, rowset_meta] : tablet->tablet_meta()->all_mutable_rs_metas()) { - rowset_meta->tablet_schema()->set_enable_single_replica_compaction( - tablet_meta_info.enable_single_replica_compaction); - } - tablet->tablet_schema_unlocked()->set_enable_single_replica_compaction( - tablet_meta_info.enable_single_replica_compaction); - need_to_save = true; - } if (tablet_meta_info.__isset.disable_auto_compaction) { std::shared_lock rlock(tablet->get_header_lock()); tablet->tablet_meta()->mutable_tablet_schema()->set_disable_auto_compaction( diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index add4afd0c998d7..d7b34574e4242e 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -413,7 +413,6 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB out->mutable_index()->CopyFrom(in.index()); out->set_version_col_idx(in.version_col_idx()); out->set_store_row_column(in.store_row_column()); - out->set_enable_single_replica_compaction(in.enable_single_replica_compaction()); out->set_skip_write_index_on_load(in.skip_write_index_on_load()); out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids()); out->set_is_dynamic_schema(in.is_dynamic_schema()); @@ -457,7 +456,6 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in) out->mutable_index()->Swap(in.mutable_index()); out->set_version_col_idx(in.version_col_idx()); out->set_store_row_column(in.store_row_column()); - out->set_enable_single_replica_compaction(in.enable_single_replica_compaction()); out->set_skip_write_index_on_load(in.skip_write_index_on_load()); out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids()); out->set_is_dynamic_schema(in.is_dynamic_schema()); @@ -514,7 +512,6 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB out->mutable_index()->CopyFrom(in.index()); out->set_version_col_idx(in.version_col_idx()); out->set_store_row_column(in.store_row_column()); - out->set_enable_single_replica_compaction(in.enable_single_replica_compaction()); out->set_skip_write_index_on_load(in.skip_write_index_on_load()); out->mutable_cluster_key_uids()->CopyFrom(in.cluster_key_uids()); out->set_is_dynamic_schema(in.is_dynamic_schema()); @@ -559,7 +556,6 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in) out->mutable_index()->Swap(in.mutable_index()); out->set_version_col_idx(in.version_col_idx()); out->set_store_row_column(in.store_row_column()); - out->set_enable_single_replica_compaction(in.enable_single_replica_compaction()); out->set_skip_write_index_on_load(in.skip_write_index_on_load()); out->mutable_cluster_key_uids()->Swap(in.mutable_cluster_key_uids()); out->set_is_dynamic_schema(in.is_dynamic_schema()); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 68151e90ed7e08..5e85a1e1cb6d85 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -466,7 +466,6 @@ DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760"); // This config can be set to limit thread number in compaction thread pool. DEFINE_mInt32(max_base_compaction_threads, "4"); DEFINE_mInt32(max_cumu_compaction_threads, "-1"); -DEFINE_mInt32(max_single_replica_compaction_threads, "-1"); DEFINE_Bool(enable_base_compaction_idle_sched, "true"); DEFINE_mInt64(base_compaction_min_rowset_num, "5"); @@ -515,9 +514,6 @@ DEFINE_mInt64(total_permits_for_compaction_score, "1000000"); // sleep interval in ms after generated compaction tasks DEFINE_mInt32(generate_compaction_tasks_interval_ms, "100"); -// sleep interval in second after update replica infos -DEFINE_mInt32(update_replica_infos_interval_seconds, "60"); - // Compaction task number per disk. // Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each. DEFINE_mInt32(compaction_task_num_per_disk, "4"); diff --git a/be/src/common/config.h b/be/src/common/config.h index bef687f70016c8..7935823041bdff 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -537,7 +537,6 @@ DECLARE_mInt32(ordered_data_compaction_min_segment_size); // This config can be set to limit thread number in compaction thread pool. DECLARE_mInt32(max_base_compaction_threads); DECLARE_mInt32(max_cumu_compaction_threads); -DECLARE_mInt32(max_single_replica_compaction_threads); DECLARE_Bool(enable_base_compaction_idle_sched); DECLARE_mInt64(base_compaction_min_rowset_num); @@ -584,8 +583,6 @@ DECLARE_mInt64(total_permits_for_compaction_score); // sleep interval in ms after generated compaction tasks DECLARE_mInt32(generate_compaction_tasks_interval_ms); -// sleep interval in second after update replica infos -DECLARE_mInt32(update_replica_infos_interval_seconds); // Compaction task number per disk. // Must be greater than 2, because Base compaction and Cumulative compaction have at least one thread each. diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index 81f3469dc2ce03..dedecbee0389d3 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -79,9 +79,6 @@ DEFINE_ENGINE_COUNTER_METRIC(finish_task_requests_total, finish_task, total); DEFINE_ENGINE_COUNTER_METRIC(finish_task_requests_failed, finish_task, failed); DEFINE_ENGINE_COUNTER_METRIC(base_compaction_request_total, base_compaction, total); DEFINE_ENGINE_COUNTER_METRIC(base_compaction_request_failed, base_compaction, failed); -DEFINE_ENGINE_COUNTER_METRIC(single_compaction_request_total, single_compaction, total); -DEFINE_ENGINE_COUNTER_METRIC(single_compaction_request_failed, single_compaction, failed); -DEFINE_ENGINE_COUNTER_METRIC(single_compaction_request_cancelled, single_compaction, cancelled); DEFINE_ENGINE_COUNTER_METRIC(cumulative_compaction_request_total, cumulative_compaction, total); DEFINE_ENGINE_COUNTER_METRIC(cumulative_compaction_request_failed, cumulative_compaction, failed); DEFINE_ENGINE_COUNTER_METRIC(publish_task_request_total, publish, total); @@ -309,9 +306,6 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, base_compaction_request_failed); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, cumulative_compaction_request_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, cumulative_compaction_request_failed); - INT_COUNTER_METRIC_REGISTER(_server_metric_entity, single_compaction_request_total); - INT_COUNTER_METRIC_REGISTER(_server_metric_entity, single_compaction_request_failed); - INT_COUNTER_METRIC_REGISTER(_server_metric_entity, single_compaction_request_cancelled); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, publish_task_request_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, publish_task_failed_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, alter_inverted_index_requests_total); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index faa4830c2b1b81..9638164f4dafc5 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -88,9 +88,6 @@ class DorisMetrics { IntCounter* base_compaction_request_failed = nullptr; IntCounter* cumulative_compaction_request_total = nullptr; IntCounter* cumulative_compaction_request_failed = nullptr; - IntCounter* single_compaction_request_total = nullptr; - IntCounter* single_compaction_request_failed = nullptr; - IntCounter* single_compaction_request_cancelled = nullptr; IntCounter* local_compaction_read_rows_total = nullptr; IntCounter* local_compaction_read_bytes_total = nullptr; diff --git a/be/src/service/http/action/compaction_action.cpp b/be/src/service/http/action/compaction_action.cpp index 975b822fa9f09f..187c3fe513d117 100644 --- a/be/src/service/http/action/compaction_action.cpp +++ b/be/src/service/http/action/compaction_action.cpp @@ -40,7 +40,6 @@ #include "storage/compaction/cumulative_compaction.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" -#include "storage/compaction/single_replica_compaction.h" #include "storage/compaction_task_tracker.h" #include "storage/olap_define.h" #include "storage/storage_engine.h" @@ -140,15 +139,6 @@ Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j return Status::NotSupported("The compaction type '{}' is not supported", compaction_type); } - // "remote" = "true" means tablet should do single replica compaction to fetch rowset from peer - bool fetch_from_remote = false; - std::string param_remote = req->param(PARAM_COMPACTION_REMOTE); - if (param_remote == "true") { - fetch_from_remote = true; - } else if (!param_remote.empty() && param_remote != "false") { - return Status::NotSupported("The remote = '{}' is not supported", param_remote); - } - // "force" = "true" means skip permit limiter when submitting full compaction to thread pool bool force = false; std::string param_force = req->param(PARAM_COMPACTION_FORCE); @@ -173,9 +163,6 @@ Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); } - if (fetch_from_remote && !tablet->should_fetch_from_peer()) { - return Status::NotSupported("tablet should do compaction locally"); - } DBUG_EXECUTE_IF("CompactionAction._handle_run_compaction.submit_cumu_task", { RETURN_IF_ERROR(_engine.submit_compaction_task( tablet, CompactionType::CUMULATIVE_COMPACTION, false)); @@ -193,8 +180,8 @@ Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j force, true, 1)); } else { // 3. execute base/cumulative compaction task in a detached thread - std::packaged_task task([this, tablet, compaction_type, fetch_from_remote]() { - return _execute_compaction_callback(tablet, compaction_type, fetch_from_remote); + std::packaged_task task([this, tablet, compaction_type]() { + return _execute_compaction_callback(tablet, compaction_type); }); std::future future_obj = task.get_future(); std::thread(std::move(task)).detach(); @@ -303,8 +290,7 @@ Status CompactionAction::_handle_run_status_compaction(HttpRequest* req, std::st } Status CompactionAction::_execute_compaction_callback(TabletSharedPtr tablet, - const std::string& compaction_type, - bool fetch_from_remote) { + const std::string& compaction_type) { MonotonicStopWatch timer; timer.start(); @@ -362,15 +348,7 @@ Status CompactionAction::_execute_compaction_callback(TabletSharedPtr tablet, } } } else if (compaction_type == PARAM_COMPACTION_CUMULATIVE) { - if (fetch_from_remote) { - SingleReplicaCompaction single_compaction(_engine, tablet, - CompactionType::CUMULATIVE_COMPACTION); - res = do_compact(single_compaction, CompactionProfileType::CUMULATIVE); - if (!res) { - LOG(WARNING) << "failed to do single compaction. res=" << res - << ", table=" << tablet->tablet_id(); - } - } else { + { CumulativeCompaction cumulative_compaction(_engine, tablet); res = do_compact(cumulative_compaction, CompactionProfileType::CUMULATIVE); if (!res) { diff --git a/be/src/service/http/action/compaction_action.h b/be/src/service/http/action/compaction_action.h index fd00dc9b247b8b..901fec339373c7 100644 --- a/be/src/service/http/action/compaction_action.h +++ b/be/src/service/http/action/compaction_action.h @@ -40,7 +40,6 @@ const std::string PARAM_COMPACTION_TYPE = "compact_type"; const std::string PARAM_COMPACTION_BASE = "base"; const std::string PARAM_COMPACTION_CUMULATIVE = "cumulative"; const std::string PARAM_COMPACTION_FULL = "full"; -const std::string PARAM_COMPACTION_REMOTE = "remote"; const std::string PARAM_COMPACTION_FORCE = "force"; /// This action is used for viewing the compaction status. @@ -62,8 +61,7 @@ class CompactionAction : public HttpHandlerWithAuth { Status _handle_run_compaction(HttpRequest* req, std::string* json_result); /// thread callback function for the tablet to do base/cumulative compaction - Status _execute_compaction_callback(TabletSharedPtr tablet, const std::string& compaction_type, - bool fetch_from_remote); + Status _execute_compaction_callback(TabletSharedPtr tablet, const std::string& compaction_type); /// fetch compaction running status Status _handle_run_status_compaction(HttpRequest* req, std::string* json_result); diff --git a/be/src/storage/compaction/single_replica_compaction.cpp b/be/src/storage/compaction/single_replica_compaction.cpp deleted file mode 100644 index a0b36cd417851f..00000000000000 --- a/be/src/storage/compaction/single_replica_compaction.cpp +++ /dev/null @@ -1,589 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/single_replica_compaction.h" - -#include -#include -#include -#include - -#include "common/logging.h" -#include "common/metrics/doris_metrics.h" -#include "io/fs/file_system.h" -#include "io/fs/local_file_system.h" -#include "io/fs/path.h" -#include "runtime/exec_env.h" -#include "runtime/memory/mem_tracker_limiter.h" -#include "service/brpc.h" -#include "service/http/http_client.h" -#include "storage/rowset/rowset_factory.h" -#include "storage/rowset/rowset_meta.h" -#include "storage/snapshot/snapshot_manager.h" -#include "storage/storage_engine.h" -#include "storage/tablet/tablet_meta.h" -#include "storage/task/engine_clone_task.h" -#include "util/brpc_client_cache.h" -#include "util/client_cache.h" -#include "util/security.h" -#include "util/thrift_rpc_helper.h" -#include "util/trace.h" - -namespace doris { -using namespace ErrorCode; - -SingleReplicaCompaction::SingleReplicaCompaction(StorageEngine& engine, - const TabletSharedPtr& tablet, - CompactionType compaction_type) - : CompactionMixin(engine, tablet, - "SingleReplicaCompaction:" + std::to_string(tablet->tablet_id())), - _compaction_type(compaction_type) {} - -SingleReplicaCompaction::~SingleReplicaCompaction() = default; - -Status SingleReplicaCompaction::prepare_compact() { - VLOG_CRITICAL << _tablet->tablet_id() << " prepare single replcia compaction and pick rowsets!"; - if (!tablet()->init_succeeded()) { - return Status::Error("_tablet init failed"); - } - - // Single replica compaction does not require picking _input_rowsets - // _input_rowsets depends on the fetched _output_version - return Status::OK(); -} - -Status SingleReplicaCompaction::execute_compact() { - if (!tablet()->should_fetch_from_peer()) { - return Status::Aborted("compaction should be performed locally"); - } - std::unique_lock lock_cumu(tablet()->get_cumulative_compaction_lock(), - std::try_to_lock); - if (!lock_cumu.owns_lock()) { - return Status::Error( - "The tablet is under cumulative compaction. tablet={}", _tablet->tablet_id()); - } - - std::unique_lock lock_base(tablet()->get_base_compaction_lock(), std::try_to_lock); - if (!lock_base.owns_lock()) { - return Status::Error( - "another base compaction is running. tablet={}", _tablet->tablet_id()); - } - - SCOPED_ATTACH_TASK(_mem_tracker); - - // do single replica compaction - RETURN_IF_ERROR(_do_single_replica_compaction()); - - _state = CompactionState::SUCCESS; - - return Status::OK(); -} - -Status SingleReplicaCompaction::_do_single_replica_compaction() { - tablet()->data_dir()->disks_compaction_num_increment(1); - Status st = _do_single_replica_compaction_impl(); - tablet()->data_dir()->disks_compaction_num_increment(-1); - - return st; -} - -Status SingleReplicaCompaction::_do_single_replica_compaction_impl() { - DBUG_EXECUTE_IF("do_single_compaction_return_ok", { return Status::OK(); }); - TReplicaInfo addr; - std::string token; - // 1. get peer replica info - DBUG_EXECUTE_IF("single_compaction_failed_get_peer", - { return Status::Aborted("tablet don't have peer replica"); }); - if (!_engine.get_peer_replica_info(_tablet->tablet_id(), &addr, &token)) { - LOG(WARNING) << _tablet->tablet_id() << " tablet don't have peer replica"; - return Status::Aborted("tablet don't have peer replica"); - } - - // 2. get verisons from peer - std::vector peer_versions; - RETURN_IF_ERROR(_get_rowset_verisons_from_peer(addr, &peer_versions)); - - Version proper_version; - // 3. find proper version to fetch - if (!_find_rowset_to_fetch(peer_versions, &proper_version)) { - return Status::Cancelled("no matched versions for single replica compaction"); - } - - // 4. fetch compaction result - RETURN_IF_ERROR(_fetch_rowset(addr, token, proper_version)); - // 5. modify rowsets in memory - RETURN_IF_ERROR(modify_rowsets()); - - // 6. update last success compaction time - if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION) { - tablet()->set_last_cumu_compaction_success_time(UnixMillis()); - } else if (compaction_type() == ReaderType::READER_BASE_COMPACTION) { - tablet()->set_last_base_compaction_success_time(UnixMillis()); - } else if (compaction_type() == ReaderType::READER_FULL_COMPACTION) { - tablet()->set_last_full_compaction_success_time(UnixMillis()); - } - - tablet()->set_last_fetched_version(_output_rowset->version()); - - int64_t current_max_version = -1; - { - std::shared_lock rdlock(_tablet->get_header_lock()); - if (RowsetSharedPtr max_rowset = _tablet->get_rowset_with_max_version()) { - current_max_version = max_rowset->end_version(); - } - } - - LOG(INFO) << "succeed to do single replica compaction" - << ". tablet=" << _tablet->tablet_id() << ", output_version=" << _output_version - << ", current_max_version=" << current_max_version - << ", input_rowsets_data_size=" << _input_rowsets_data_size - << ", input_rowsets_index_size=" << _input_rowsets_index_size - << ", input_rowsets_total_size=" << _input_rowsets_total_size - << ", input_row_num=" << _input_row_num - << ", input_segments_num=" << _input_num_segments - << ", _input_index_size=" << _input_rowsets_index_size - << ", output_rowset_data_size=" << _output_rowset->data_disk_size() - << ", output_rowset_index_size=" << _output_rowset->index_disk_size() - << ", output_rowset_total_size=" << _output_rowset->total_disk_size() - << ", output_row_num=" << _output_rowset->num_rows() - << ", output_segments_num=" << _output_rowset->num_segments(); - return Status::OK(); -} - -Status SingleReplicaCompaction::_get_rowset_verisons_from_peer( - const TReplicaInfo& addr, std::vector* peer_versions) { - DBUG_EXECUTE_IF("single_compaction_failed_get_peer_versions", - { return Status::Aborted("tablet failed get peer versions"); }); - PGetTabletVersionsRequest request; - request.set_tablet_id(_tablet->tablet_id()); - PGetTabletVersionsResponse response; - std::shared_ptr stub = - ExecEnv::GetInstance()->brpc_internal_client_cache()->get_client(addr.host, - addr.brpc_port); - if (stub == nullptr) { - return Status::Aborted("get rpc stub failed, host={}, port={}", addr.host, addr.brpc_port); - } - - brpc::Controller cntl; - stub->get_tablet_rowset_versions(&cntl, &request, &response, nullptr); - if (cntl.Failed()) { - return Status::Aborted("open brpc connection failed"); - } - if (response.status().status_code() != 0) { - return Status::Aborted("peer don't have tablet"); - } - if (response.versions_size() == 0) { - return Status::Aborted("no peer version"); - } - for (int i = 0; i < response.versions_size(); ++i) { - (*peer_versions).emplace_back(response.versions(i).first(), response.versions(i).second()); - } - return Status::OK(); -} - -bool SingleReplicaCompaction::_find_rowset_to_fetch(const std::vector& peer_versions, - Version* proper_version) { - // already sorted - std::vector local_versions = tablet()->get_all_local_versions(); - for (const auto& v : local_versions) { - VLOG_CRITICAL << _tablet->tablet_id() << " tablet local version: " << v.first << " - " - << v.second; - } - for (const auto& v : peer_versions) { - VLOG_CRITICAL << _tablet->tablet_id() << " tablet peer version: " << v.first << " - " - << v.second; - } - - bool find = false; - int index_peer = 0; - int index_local = 0; - // peer_versions [0-0] [1-1] [2-2] [3-5] [6-7] - // local_versions [0-0] [1-1] [2-2] [3-3] [4-4] [5-5] [6-7] - // return output_version [3-5] - // 1: skip same versions - while (index_local < local_versions.size() && index_peer < peer_versions.size()) { - if (peer_versions[index_peer].first == local_versions[index_local].first && - peer_versions[index_peer].second == local_versions[index_local].second) { - ++index_peer; - ++index_local; - continue; - } - break; - } - if (index_peer >= peer_versions.size() || index_local >= local_versions.size()) { - return false; - } - // 2: first match - if (peer_versions[index_peer].first != local_versions[index_local].first) { - return false; - } - // 3: second match - if (peer_versions[index_peer].contains(local_versions[index_local])) { - ++index_local; - while (index_local < local_versions.size()) { - if (peer_versions[index_peer].contains(local_versions[index_local])) { - ++index_local; - continue; - } - break; - } - --index_local; - if (local_versions[index_local].second == peer_versions[index_peer].second) { - *proper_version = peer_versions[index_peer]; - find = true; - } - } - if (find) { - // 4. reset input rowsets - _input_rowsets.clear(); - tablet()->traverse_rowsets([this, &proper_version](const auto& rs) { - // only need rowset in proper_version - if (rs->is_local() && proper_version->contains(rs->version())) { - this->_input_rowsets.emplace_back(rs); - } - }); - std::sort(_input_rowsets.begin(), _input_rowsets.end(), Rowset::comparator); - DCHECK_EQ(_input_rowsets.front()->start_version(), proper_version->first); - DCHECK_EQ(_input_rowsets.back()->end_version(), proper_version->second); - if (_input_rowsets.front()->start_version() != proper_version->first || - _input_rowsets.back()->end_version() != proper_version->second) { - LOG(WARNING) << fmt::format( - "single compaction input rowsets error, tablet_id={}, input rowset = [{}-{}], " - "remote rowset = {}", - tablet()->tablet_id(), _input_rowsets.front()->start_version(), - _input_rowsets.back()->end_version(), proper_version->to_string()); - return false; - } - for (auto& rowset : _input_rowsets) { - _input_rowsets_data_size += rowset->data_disk_size(); - _input_row_num += rowset->num_rows(); - _input_num_segments += rowset->num_segments(); - _input_rowsets_index_size += rowset->index_disk_size(); - _input_rowsets_total_size += rowset->data_disk_size() + rowset->index_disk_size(); - } - _output_version = *proper_version; - } - return find; -} - -Status SingleReplicaCompaction::_fetch_rowset(const TReplicaInfo& addr, const std::string& token, - const Version& rowset_version) { - LOG(INFO) << "begin to fetch compaction result, tablet_id=" << _tablet->tablet_id() - << ", addr=" << addr.host << ", version=" << rowset_version; - std::shared_lock migration_rlock(tablet()->get_migration_lock(), std::try_to_lock); - if (!migration_rlock.owns_lock()) { - return Status::Error("got migration_rlock failed. tablet={}", - _tablet->tablet_id()); - } - - std::string local_data_path = tablet()->tablet_path() + CLONE_PREFIX; - std::string local_path = local_data_path + "/"; - std::string snapshot_path; - int timeout_s = 0; - // 1: make snapshot - RETURN_IF_ERROR(_make_snapshot(addr.host, addr.be_port, _tablet->tablet_id(), - _tablet->schema_hash(), timeout_s, rowset_version, - &snapshot_path)); - Defer defer {[&, this] { - // TODO(plat1ko): Async release snapshot - auto st = _release_snapshot(addr.host, addr.be_port, snapshot_path); - if (!st.ok()) [[unlikely]] { - LOG_WARNING("failed to release snapshot in remote BE") - .tag("host", addr.host) - .tag("port", addr.be_port) - .tag("snapshot_path", snapshot_path) - .error(st); - } - }}; - // 2: download snapshot - std::string remote_url_prefix; - { - std::stringstream ss; - ss << "http://" << addr.host << ":" << addr.http_port << HTTP_REQUEST_PREFIX - << HTTP_REQUEST_TOKEN_PARAM << token << HTTP_REQUEST_FILE_PARAM << snapshot_path << "/" - << _tablet->tablet_id() << "/" << _tablet->schema_hash() << "/"; - remote_url_prefix = ss.str(); - } - RETURN_IF_ERROR(_download_files(tablet()->data_dir(), remote_url_prefix, local_path)); - _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( - local_path, _tablet->tablet_id(), tablet()->replica_id(), _tablet->table_id(), - _tablet->partition_id(), _tablet->schema_hash())); - // 4: finish_clone: create output_rowset and link file - return _finish_clone(local_data_path, rowset_version); -} - -Status SingleReplicaCompaction::_make_snapshot(const std::string& ip, int port, TTableId tablet_id, - TSchemaHash schema_hash, int timeout_s, - const Version& version, std::string* snapshot_path) { - VLOG_NOTICE << "single replica compaction make snapshot ip=" << ip - << ", tablet_id=" << tablet_id; - TSnapshotRequest request; - request.__set_tablet_id(tablet_id); - request.__set_schema_hash(schema_hash); - request.__set_preferred_snapshot_version(g_Types_constants.TPREFER_SNAPSHOT_REQ_VERSION); - request.__set_start_version(version.first); - request.__set_end_version(version.second); - - if (timeout_s > 0) { - request.__set_timeout(timeout_s); - } - - TAgentResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( - ip, port, [&request, &result](BackendServiceConnection& client) { - client->make_snapshot(result, request); - })); - if (result.status.status_code != TStatusCode::OK) { - return Status::create(result.status); - } - - if (result.__isset.snapshot_path) { - *snapshot_path = result.snapshot_path; - if (snapshot_path->at(snapshot_path->length() - 1) != '/') { - snapshot_path->append("/"); - } - DBUG_EXECUTE_IF("single_compaction_failed_make_snapshot", - { return Status::InternalError("failed snapshot"); }); - } else { - return Status::InternalError("success snapshot without snapshot path"); - } - return Status::OK(); -} - -Status SingleReplicaCompaction::_download_files(DataDir* data_dir, - const std::string& remote_url_prefix, - const std::string& local_path) { - // Check local path exist, if exist, remove it, then create the dir - // local_file_full_path = tabletid/clone， for a specific tablet, there should be only one folder - // if this folder exists, then should remove it - // for example, BE clone from BE 1 to download file 1 with version (2,2), but clone from BE 1 failed - // then it will try to clone from BE 2, but it will find the file 1 already exist, but file 1 with same - // name may have different versions. - VLOG_DEBUG << "single replica compaction begin to download files, remote path=" - << mask_token(remote_url_prefix) << " local_path=" << local_path; - RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(local_path)); - RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(local_path)); - - // Get remote dir file list - std::string file_list_str; - auto list_files_cb = [&remote_url_prefix, &file_list_str](HttpClient* client) { - RETURN_IF_ERROR(client->init(remote_url_prefix)); - client->set_timeout_ms(LIST_REMOTE_FILE_TIMEOUT * 1000); - RETURN_IF_ERROR(client->execute(&file_list_str)); - return Status::OK(); - }; - RETURN_IF_ERROR(HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, list_files_cb)); - std::vector file_name_list = - absl::StrSplit(file_list_str, "\n", absl::SkipWhitespace()); - - // If the header file is not exist, the table couldn't loaded by olap engine. - // Avoid of data is not complete, we copy the header file at last. - // The header file's name is end of .hdr. - for (int i = 0; i < file_name_list.size() - 1; ++i) { - if (file_name_list[i].ends_with(".hdr")) { - std::swap(file_name_list[i], file_name_list[file_name_list.size() - 1]); - break; - } - } - - // Get copy from remote - uint64_t total_file_size = 0; - MonotonicStopWatch watch; - watch.start(); - auto curl = std::unique_ptr(curl_easy_init(), - &curl_easy_cleanup); - if (!curl) { - return Status::InternalError("single compaction init curl failed"); - } - for (auto& file_name : file_name_list) { - auto remote_file_url = remote_url_prefix + file_name; - - // get file length - uint64_t file_size = 0; - auto get_file_size_cb = [&remote_file_url, &file_size](HttpClient* client) { - RETURN_IF_ERROR(client->init(remote_file_url)); - client->set_timeout_ms(GET_LENGTH_TIMEOUT * 1000); - RETURN_IF_ERROR(client->head()); - RETURN_IF_ERROR(client->get_content_length(&file_size)); - return Status::OK(); - }; - RETURN_IF_ERROR( - HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, get_file_size_cb)); - // check disk capacity - if (data_dir->reach_capacity_limit(file_size)) { - return Status::Error( - "reach the capacity limit of path {}, file_size={}", data_dir->path(), - file_size); - } - - total_file_size += file_size; - uint64_t estimate_timeout = file_size / config::download_low_speed_limit_kbps / 1024; - if (estimate_timeout < config::download_low_speed_time) { - estimate_timeout = config::download_low_speed_time; - } - - std::string local_file_path = local_path + file_name; - - LOG(INFO) << "single replica compaction begin to download file from: " - << mask_token(remote_file_url) << " to: " << local_file_path - << ". size(B): " << file_size << ", timeout(s): " << estimate_timeout; - - auto download_cb = [&remote_file_url, estimate_timeout, &local_file_path, - file_size](HttpClient* client) { - RETURN_IF_ERROR(client->init(remote_file_url)); - client->set_timeout_ms(estimate_timeout * 1000); - RETURN_IF_ERROR(client->download(local_file_path)); - - DBUG_EXECUTE_IF("single_compaction_failed_download_file", - { return Status::InternalError("failed to download file"); }); - // Check file length - uint64_t local_file_size = std::filesystem::file_size(local_file_path); - if (local_file_size != file_size) { - LOG(WARNING) << "download file length error" - << ", remote_path=" << mask_token(remote_file_url) - << ", file_size=" << file_size - << ", local_file_size=" << local_file_size; - return Status::InternalError("downloaded file size is not equal"); - } - return io::global_local_filesystem()->permission(local_file_path, - io::LocalFileSystem::PERMS_OWNER_RW); - }; - RETURN_IF_ERROR(HttpClient::execute_with_retry(DOWNLOAD_FILE_MAX_RETRY, 1, download_cb)); - } // Clone files from remote backend - - uint64_t total_time_ms = watch.elapsed_time() / 1000 / 1000; - total_time_ms = total_time_ms > 0 ? total_time_ms : 0; - double copy_rate = 0.0; - if (total_time_ms > 0) { - copy_rate = total_file_size / ((double)total_time_ms) / 1000; - } - LOG(INFO) << "succeed to single replica compaction copy tablet " << _tablet->tablet_id() - << ", total file size: " << total_file_size << " B" - << ", cost: " << total_time_ms << " ms" - << ", rate: " << copy_rate << " MB/s"; - return Status::OK(); -} - -Status SingleReplicaCompaction::_release_snapshot(const std::string& ip, int port, - const std::string& snapshot_path) { - TAgentResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( - ip, port, [&snapshot_path, &result](BackendServiceConnection& client) { - client->release_snapshot(result, snapshot_path); - })); - return Status::create(result.status); -} - -Status SingleReplicaCompaction::_finish_clone(const std::string& clone_dir, - const Version& output_version) { - Status res = Status::OK(); - std::vector linked_success_files; - { - do { - // check clone dir existed - bool exists = true; - RETURN_IF_ERROR(io::global_local_filesystem()->exists(clone_dir, &exists)); - if (!exists) { - return Status::InternalError("clone dir not existed. clone_dir={}", clone_dir); - } - - // Load src header. - // The tablet meta info is downloaded from source BE as .hdr file. - // So we load it and generate cloned_tablet_meta. - auto cloned_tablet_meta_file = - fmt::format("{}/{}.hdr", clone_dir, _tablet->tablet_id()); - auto cloned_tablet_meta = std::make_shared(); - RETURN_IF_ERROR(cloned_tablet_meta->create_from_file(cloned_tablet_meta_file)); - - // remove the cloned meta file - RETURN_IF_ERROR(io::global_local_filesystem()->delete_file(cloned_tablet_meta_file)); - - RowsetMetaSharedPtr output_rs_meta = - cloned_tablet_meta->acquire_rs_meta_by_version(output_version); - if (output_rs_meta == nullptr) { - LOG(WARNING) << "version not found in cloned tablet meta when do single compaction"; - return Status::InternalError("version not found in cloned tablet meta"); - } - res = tablet()->create_rowset(output_rs_meta, &_output_rowset); - if (!res.ok()) { - LOG(WARNING) << "fail to init rowset. version=" << output_version; - return res; - } - - // check all files in /clone and /tablet - std::vector clone_files; - RETURN_IF_ERROR( - io::global_local_filesystem()->list(clone_dir, true, &clone_files, &exists)); - std::unordered_set clone_file_names; - for (auto& file : clone_files) { - clone_file_names.insert(file.file_name); - } - - std::vector local_files; - const auto& tablet_dir = tablet()->tablet_path(); - RETURN_IF_ERROR( - io::global_local_filesystem()->list(tablet_dir, true, &local_files, &exists)); - std::unordered_set local_file_names; - for (auto& file : local_files) { - local_file_names.insert(file.file_name); - } - - /// Traverse all downloaded clone files in CLONE dir. - /// If it does not exist in local tablet dir, link the file to local tablet dir - /// And save all linked files in linked_success_files. - for (const std::string& clone_file : clone_file_names) { - if (local_file_names.find(clone_file) != local_file_names.end()) { - VLOG_NOTICE << "find same file when clone, skip it. " - << "tablet=" << _tablet->tablet_id() - << ", clone_file=" << clone_file; - continue; - } - - auto from = fmt::format("{}/{}", clone_dir, clone_file); - auto to = fmt::format("{}/{}", tablet_dir, clone_file); - RETURN_IF_ERROR(io::global_local_filesystem()->link_file(from, to)); - linked_success_files.emplace_back(std::move(to)); - } - - if (!res.ok()) { - break; - } - } while (false); - - // clear linked files if errors happen - if (!res.ok()) { - std::vector paths; - for (auto& file : linked_success_files) { - paths.emplace_back(file); - } - RETURN_IF_ERROR(io::global_local_filesystem()->batch_delete(paths)); - } - } - // clear clone dir - std::filesystem::path clone_dir_path(clone_dir); - std::error_code ec; - std::filesystem::remove_all(clone_dir_path, ec); - if (ec) { - LOG(WARNING) << "failed to remove=" << clone_dir_path << " msg=" << ec.message(); - return Status::IOError("failed to remove {}, due to {}", clone_dir, ec.message()); - } - LOG(INFO) << "finish to clone data, clear downloaded data. res=" << res - << ", tablet=" << _tablet->tablet_id() << ", clone_dir=" << clone_dir; - return res; -} - -} // namespace doris diff --git a/be/src/storage/compaction/single_replica_compaction.h b/be/src/storage/compaction/single_replica_compaction.h deleted file mode 100644 index c4f4ee0b15e55e..00000000000000 --- a/be/src/storage/compaction/single_replica_compaction.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "common/status.h" -#include "storage/compaction/compaction.h" -#include "storage/rowset/pending_rowset_helper.h" - -namespace doris { - -class DataDir; - -// SingleReplicaCompaction is used to fetch peer replica compaction result. -class SingleReplicaCompaction final : public CompactionMixin { -public: - SingleReplicaCompaction(StorageEngine& engine, const TabletSharedPtr& tablet, - CompactionType compaction_type); - ~SingleReplicaCompaction() override; - - Status prepare_compact() override; - Status execute_compact() override; - - inline CompactionType real_compact_type() const { return _compaction_type; } - -protected: - std::string_view compaction_name() const override { return "single replica compaction"; } - ReaderType compaction_type() const override { - return (_compaction_type == CompactionType::CUMULATIVE_COMPACTION) - ? ReaderType::READER_CUMULATIVE_COMPACTION - : ReaderType::READER_BASE_COMPACTION; - } - -private: - Status _do_single_replica_compaction(); - Status _do_single_replica_compaction_impl(); - bool _find_rowset_to_fetch(const std::vector& peer_versions, Version* proper_version); - Status _get_rowset_verisons_from_peer(const TReplicaInfo& addr, - std::vector* peer_versions); - Status _fetch_rowset(const TReplicaInfo& addr, const std::string& token, - const Version& version); - Status _make_snapshot(const std::string& ip, int port, TTableId tablet_id, - TSchemaHash schema_hash, int timeout_s, const Version& version, - std::string* snapshot_path); - Status _download_files(DataDir* data_dir, const std::string& remote_url_prefix, - const std::string& local_path); - Status _release_snapshot(const std::string& ip, int port, const std::string& snapshot_path); - Status _finish_clone(const std::string& clone_dir, const Version& version); - CompactionType _compaction_type; - - std::vector _pending_rs_guards; -}; - -} // namespace doris \ No newline at end of file diff --git a/be/src/storage/olap_server.cpp b/be/src/storage/olap_server.cpp index 07b840bf5cb6ce..3613b6101cbbba 100644 --- a/be/src/storage/olap_server.cpp +++ b/be/src/storage/olap_server.cpp @@ -26,7 +26,6 @@ #include #include // IWYU pragma: no_include -#include #include #include // IWYU pragma: keep @@ -64,7 +63,6 @@ #include "storage/compaction/cumulative_compaction.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" -#include "storage/compaction/single_replica_compaction.h" #include "storage/compaction_task_tracker.h" #include "storage/data_dir.h" #include "storage/olap_common.h" @@ -87,7 +85,6 @@ #include "util/mem_info.h" #include "util/thread.h" #include "util/threadpool.h" -#include "util/thrift_rpc_helper.h" #include "util/time.h" #include "util/uid_util.h" #include "util/work_thread_pool.hpp" @@ -103,9 +100,6 @@ volatile uint32_t g_schema_change_active_threads = 0; bvar::Status g_cumu_compaction_task_num_per_round("cumu_compaction_task_num_per_round", 0); bvar::Status g_base_compaction_task_num_per_round("base_compaction_task_num_per_round", 0); -static const uint64_t DEFAULT_SEED = 104729; -static const uint64_t MOD_PRIME = 7652413; - CompactionSubmitRegistry::CompactionSubmitRegistry(CompactionSubmitRegistry&& r) { std::swap(_tablet_submitted_cumu_compaction, r._tablet_submitted_cumu_compaction); std::swap(_tablet_submitted_base_compaction, r._tablet_submitted_base_compaction); @@ -208,15 +202,6 @@ static int32_t get_base_compaction_threads_num(size_t data_dirs_num) { return threads_num; } -static int32_t get_single_replica_compaction_threads_num(size_t data_dirs_num) { - int32_t threads_num = config::max_single_replica_compaction_threads; - if (threads_num == -1) { - threads_num = cast_set(data_dirs_num); - } - threads_num = threads_num <= 0 ? 1 : threads_num; - return threads_num; -} - Status StorageEngine::start_bg_threads(std::shared_ptr wg_sptr) { RETURN_IF_ERROR(Thread::create( "StorageEngine", "unused_rowset_monitor_thread", @@ -247,8 +232,6 @@ Status StorageEngine::start_bg_threads(std::shared_ptr wg_sptr) { auto base_compaction_threads = get_base_compaction_threads_num(data_dirs.size()); auto cumu_compaction_threads = get_cumu_compaction_threads_num(data_dirs.size()); - auto single_replica_compaction_threads = - get_single_replica_compaction_threads_num(data_dirs.size()); RETURN_IF_ERROR(ThreadPoolBuilder("BaseCompactionTaskThreadPool") .set_min_threads(base_compaction_threads) @@ -258,10 +241,6 @@ Status StorageEngine::start_bg_threads(std::shared_ptr wg_sptr) { .set_min_threads(cumu_compaction_threads) .set_max_threads(cumu_compaction_threads) .build(&_cumu_compaction_thread_pool)); - RETURN_IF_ERROR(ThreadPoolBuilder("SingleReplicaCompactionTaskThreadPool") - .set_min_threads(single_replica_compaction_threads) - .set_max_threads(single_replica_compaction_threads) - .build(&_single_replica_compaction_thread_pool)); if (config::enable_segcompaction) { RETURN_IF_ERROR(ThreadPoolBuilder("SegCompactionTaskThreadPool") @@ -281,11 +260,6 @@ Status StorageEngine::start_bg_threads(std::shared_ptr wg_sptr) { &_compaction_tasks_producer_thread)); LOG(INFO) << "compaction tasks producer thread started"; - RETURN_IF_ERROR(Thread::create( - "StorageEngine", "_update_replica_infos_thread", - [this]() { this->_update_replica_infos_callback(); }, &_update_replica_infos_thread)); - LOG(INFO) << "tablet replicas info update thread started"; - int32_t max_checkpoint_thread_num = config::max_meta_checkpoint_threads; if (max_checkpoint_thread_num < 0) { max_checkpoint_thread_num = cast_set(data_dirs.size()); @@ -623,29 +597,6 @@ void StorageEngine::_adjust_compaction_thread_num() { << " to " << cumu_compaction_threads_num; } } - - auto single_replica_compaction_threads_num = - get_single_replica_compaction_threads_num(_store_map.size()); - if (_single_replica_compaction_thread_pool->max_threads() != - single_replica_compaction_threads_num) { - int old_max_threads = _single_replica_compaction_thread_pool->max_threads(); - Status status = _single_replica_compaction_thread_pool->set_max_threads( - single_replica_compaction_threads_num); - if (status.ok()) { - VLOG_NOTICE << "update single replica compaction thread pool max_threads from " - << old_max_threads << " to " << single_replica_compaction_threads_num; - } - } - if (_single_replica_compaction_thread_pool->min_threads() != - single_replica_compaction_threads_num) { - int old_min_threads = _single_replica_compaction_thread_pool->min_threads(); - Status status = _single_replica_compaction_thread_pool->set_min_threads( - single_replica_compaction_threads_num); - if (status.ok()) { - VLOG_NOTICE << "update single replica compaction thread pool min_threads from " - << old_min_threads << " to " << single_replica_compaction_threads_num; - } - } } void StorageEngine::_compaction_tasks_producer_callback() { @@ -762,145 +713,6 @@ void StorageEngine::_compaction_tasks_producer_callback() { } while (!_stop_background_threads_latch.wait_for(std::chrono::milliseconds(interval))); } -void StorageEngine::_update_replica_infos_callback() { -#ifdef GOOGLE_PROFILER - ProfilerRegisterThread(); -#endif - LOG(INFO) << "start to update replica infos!"; - - int64_t interval = config::update_replica_infos_interval_seconds; - do { - auto all_tablets = _tablet_manager->get_all_tablet([](Tablet* t) { - return t->is_used() && t->tablet_state() == TABLET_RUNNING && - !t->tablet_meta()->tablet_schema()->disable_auto_compaction() && - t->tablet_meta()->tablet_schema()->enable_single_replica_compaction(); - }); - ClusterInfo* cluster_info = ExecEnv::GetInstance()->cluster_info(); - if (cluster_info == nullptr) { - LOG(WARNING) << "Have not get FE Master heartbeat yet"; - std::this_thread::sleep_for(std::chrono::seconds(2)); - continue; - } - TNetworkAddress master_addr = cluster_info->master_fe_addr; - if (master_addr.hostname == "" || master_addr.port == 0) { - LOG(WARNING) << "Have not get FE Master heartbeat yet"; - std::this_thread::sleep_for(std::chrono::seconds(2)); - continue; - } - - int start = 0; - int tablet_size = cast_set(all_tablets.size()); - // The while loop may take a long time, we should skip it when stop - while (start < tablet_size && _stop_background_threads_latch.count() > 0) { - int batch_size = std::min(100, tablet_size - start); - int end = start + batch_size; - TGetTabletReplicaInfosRequest request; - TGetTabletReplicaInfosResult result; - for (int i = start; i < end; i++) { - request.tablet_ids.emplace_back(all_tablets[i]->tablet_id()); - } - Status rpc_st = ThriftRpcHelper::rpc( - master_addr.hostname, master_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->getTabletReplicaInfos(result, request); - }); - - if (!rpc_st.ok()) { - LOG(WARNING) << "Failed to get tablet replica infos, encounter rpc failure, " - "tablet start: " - << start << " end: " << end; - continue; - } - - std::unique_lock lock(_peer_replica_infos_mutex); - for (const auto& it : result.tablet_replica_infos) { - auto tablet_id = it.first; - auto tablet = _tablet_manager->get_tablet(tablet_id); - if (tablet == nullptr) { - VLOG_CRITICAL << "tablet ptr is nullptr"; - continue; - } - - VLOG_NOTICE << tablet_id << " tablet has " << it.second.size() << " replicas"; - uint64_t min_modulo = MOD_PRIME; - TReplicaInfo peer_replica; - for (const auto& replica : it.second) { - int64_t peer_replica_id = replica.replica_id; - uint64_t modulo = HashUtil::hash64(&peer_replica_id, sizeof(peer_replica_id), - DEFAULT_SEED) % - MOD_PRIME; - if (modulo < min_modulo) { - peer_replica = replica; - min_modulo = modulo; - } - } - VLOG_NOTICE << "tablet " << tablet_id << ", peer replica host is " - << peer_replica.host; - _peer_replica_infos[tablet_id] = peer_replica; - } - _token = result.token; - VLOG_NOTICE << "get tablet replica infos from fe, size is " << end - start - << " token = " << result.token; - start = end; - } - interval = config::update_replica_infos_interval_seconds; - } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))); -} - -Status StorageEngine::_submit_single_replica_compaction_task(TabletSharedPtr tablet, - CompactionType compaction_type) { - // For single replica compaction, the local version to be merged is determined based on the version fetched from the peer replica. - // Therefore, it is currently not possible to determine whether it should be a base compaction or cumulative compaction. - // As a result, the tablet needs to be pushed to both the _tablet_submitted_cumu_compaction and the _tablet_submitted_base_compaction simultaneously. - bool already_exist = - _compaction_submit_registry.insert(tablet, CompactionType::CUMULATIVE_COMPACTION); - if (already_exist) { - return Status::AlreadyExist( - "compaction task has already been submitted, tablet_id={}", tablet->tablet_id()); - } - - already_exist = _compaction_submit_registry.insert(tablet, CompactionType::BASE_COMPACTION); - if (already_exist) { - _pop_tablet_from_submitted_compaction(tablet, CompactionType::CUMULATIVE_COMPACTION); - return Status::AlreadyExist( - "compaction task has already been submitted, tablet_id={}", tablet->tablet_id()); - } - - auto compaction = std::make_shared(*this, tablet, compaction_type); - DorisMetrics::instance()->single_compaction_request_total->increment(1); - auto st = compaction->prepare_compact(); - - auto clean_single_replica_compaction = [tablet, this]() { - _pop_tablet_from_submitted_compaction(tablet, CompactionType::CUMULATIVE_COMPACTION); - _pop_tablet_from_submitted_compaction(tablet, CompactionType::BASE_COMPACTION); - }; - - if (!st.ok()) { - clean_single_replica_compaction(); - if (!st.is()) { - LOG(WARNING) << "failed to prepare single replica compaction, tablet_id=" - << tablet->tablet_id() << " : " << st; - return st; - } - return Status::OK(); // No suitable version, regard as OK - } - - auto submit_st = _single_replica_compaction_thread_pool->submit_func( - [tablet, compaction = std::move(compaction), - clean_single_replica_compaction]() mutable { - tablet->execute_single_replica_compaction(*compaction); - clean_single_replica_compaction(); - }); - if (!submit_st.ok()) { - clean_single_replica_compaction(); - return Status::InternalError( - "failed to submit single replica compaction task to thread pool, " - "tablet_id={}", - tablet->tablet_id()); - } - return Status::OK(); -} - void StorageEngine::get_tablet_rowset_versions(const PGetTabletVersionsRequest* request, PGetTabletVersionsResponse* response) { TabletSharedPtr tablet = _tablet_manager->get_tablet(request->tablet_id()); @@ -1059,18 +871,6 @@ void StorageEngine::_pop_tablet_from_submitted_compaction(TabletSharedPtr tablet Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, CompactionType compaction_type, bool force, int trigger_method) { - if (tablet->tablet_meta()->tablet_schema()->enable_single_replica_compaction() && - should_fetch_from_peer(tablet->tablet_id())) { - VLOG_CRITICAL << "start to submit single replica compaction task for tablet: " - << tablet->tablet_id(); - Status st = _submit_single_replica_compaction_task(tablet, compaction_type); - if (!st.ok()) { - LOG(WARNING) << "failed to submit single replica compaction task for tablet: " - << tablet->tablet_id() << ", err: " << st; - } - - return Status::OK(); - } bool already_exist = _compaction_submit_registry.insert(tablet, compaction_type); if (already_exist) { return Status::AlreadyExist( diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp index 5e79d8f8dcf37e..8eb1cd77c0a917 100644 --- a/be/src/storage/storage_engine.cpp +++ b/be/src/storage/storage_engine.cpp @@ -61,7 +61,6 @@ #include "load/stream_load/stream_load_recorder.h" #include "runtime/exec_env.h" #include "storage/binlog.h" -#include "storage/compaction/single_replica_compaction.h" #include "storage/data_dir.h" #include "storage/id_manager.h" #include "storage/olap_common.h" @@ -744,7 +743,6 @@ void StorageEngine::stop() { } THREAD_JOIN(_compaction_tasks_producer_thread); - THREAD_JOIN(_update_replica_infos_thread); THREAD_JOIN(_unused_rowset_monitor_thread); THREAD_JOIN(_garbage_sweeper_thread); THREAD_JOIN(_disk_stat_monitor_thread); @@ -772,9 +770,6 @@ void StorageEngine::stop() { if (_cumu_compaction_thread_pool) { _cumu_compaction_thread_pool->shutdown(); } - if (_single_replica_compaction_thread_pool) { - _single_replica_compaction_thread_pool->shutdown(); - } if (_seg_compaction_thread_pool) { _seg_compaction_thread_pool->shutdown(); @@ -1542,23 +1537,6 @@ PendingRowsetGuard StorageEngine::add_pending_rowset(const RowsetWriterContext& return _pending_remote_rowsets.add(ctx.rowset_id); } -bool StorageEngine::get_peer_replica_info(int64_t tablet_id, TReplicaInfo* replica, - std::string* token) { - TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); - if (tablet == nullptr) { - LOG(WARNING) << "tablet is no longer exist: tablet_id=" << tablet_id; - return false; - } - std::unique_lock lock(_peer_replica_infos_mutex); - if (_peer_replica_infos.contains(tablet_id) && - _peer_replica_infos[tablet_id].replica_id != tablet->replica_id()) { - *replica = _peer_replica_infos[tablet_id]; - *token = _token; - return true; - } - return false; -} - bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vector* backends) { TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); if (tablet == nullptr) { @@ -1599,7 +1577,6 @@ bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vector lock(_peer_replica_infos_mutex); if (result.tablet_replica_infos.contains(tablet_id)) { std::vector reps = result.tablet_replica_infos[tablet_id]; if (reps.empty()) [[unlikely]] { @@ -1637,25 +1614,6 @@ bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vectorget_tablet(tablet_id); - if (tablet == nullptr) { - LOG(WARNING) << "tablet is no longer exist: tablet_id=" << tablet_id; - return false; - } - std::unique_lock lock(_peer_replica_infos_mutex); - if (_peer_replica_infos.contains(tablet_id)) { - return _peer_replica_infos[tablet_id].replica_id != tablet->replica_id(); - } - return false; -} - // Return json: // { // "CumulativeCompaction": { diff --git a/be/src/storage/storage_engine.h b/be/src/storage/storage_engine.h index 57b087211793a0..29f088ca8312b1 100644 --- a/be/src/storage/storage_engine.h +++ b/be/src/storage/storage_engine.h @@ -61,7 +61,6 @@ class MemTableFlushExecutor; class SegcompactionWorker; class BaseCompaction; class CumulativeCompaction; -class SingleReplicaCompaction; class CumulativeCompactionPolicy; class StreamLoadRecorder; class TCloneReq; @@ -342,12 +341,8 @@ class StorageEngine final : public BaseStorageEngine { void get_tablet_rowset_versions(const PGetTabletVersionsRequest* request, PGetTabletVersionsResponse* response); - bool get_peer_replica_info(int64_t tablet_id, TReplicaInfo* replica, std::string* token); - bool get_peers_replica_backends(int64_t tablet_id, std::vector* backends); - bool should_fetch_from_peer(int64_t tablet_id); - const std::shared_ptr& get_stream_load_recorder() { return _stream_load_recorder; } @@ -440,8 +435,6 @@ class StorageEngine final : public BaseStorageEngine { void _compaction_tasks_producer_callback(); - void _update_replica_infos_callback(); - std::vector _generate_compaction_tasks(CompactionType compaction_type, std::vector& data_dirs, bool check_score); @@ -457,9 +450,6 @@ class StorageEngine final : public BaseStorageEngine { CompactionType compaction_type, int64_t permits, bool force, int64_t compaction_id = 0); - Status _submit_single_replica_compaction_task(TabletSharedPtr tablet, - CompactionType compaction_type); - void _adjust_compaction_thread_num(); void _cooldown_tasks_producer_callback(); @@ -525,7 +515,6 @@ class StorageEngine final : public BaseStorageEngine { std::shared_ptr _disk_stat_monitor_thread; // thread to produce both base and cumulative compaction tasks std::shared_ptr _compaction_tasks_producer_thread; - std::shared_ptr _update_replica_infos_thread; std::shared_ptr _cache_clean_thread; // threads to clean all file descriptor not actively in use std::vector> _path_gc_threads; @@ -545,8 +534,6 @@ class StorageEngine final : public BaseStorageEngine { // Type of new loaded data RowsetTypePB _default_rowset_type; - std::unique_ptr _single_replica_compaction_thread_pool; - std::unique_ptr _seg_compaction_thread_pool; std::unique_ptr _cold_data_compaction_thread_pool; @@ -561,11 +548,6 @@ class StorageEngine final : public BaseStorageEngine { std::mutex _low_priority_task_nums_mutex; std::unordered_map _low_priority_task_nums; - std::mutex _peer_replica_infos_mutex; - // key: tabletId - std::unordered_map _peer_replica_infos; - std::string _token; - std::atomic _wakeup_producer_flag {0}; std::mutex _compaction_producer_sleep_mutex; diff --git a/be/src/storage/tablet/tablet.cpp b/be/src/storage/tablet/tablet.cpp index a7df4d49959375..31d7e70dda34df 100644 --- a/be/src/storage/tablet/tablet.cpp +++ b/be/src/storage/tablet/tablet.cpp @@ -85,7 +85,6 @@ #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction/full_compaction.h" -#include "storage/compaction/single_replica_compaction.h" #include "storage/delete/delete_bitmap_calculator.h" #include "storage/index/indexed_column_reader.h" #include "storage/index/primary_key_index.h" @@ -1412,33 +1411,6 @@ void Tablet::get_compaction_status(std::string* json_result) { // "last failure status": "", // "last fetched rowset": "[8-10]" // } - rapidjson::Document status; - status.SetObject(); - TReplicaInfo replica_info; - std::string dummp_token; - if (tablet_meta()->tablet_schema()->enable_single_replica_compaction() && - _engine.get_peer_replica_info(tablet_id(), &replica_info, &dummp_token)) { - // remote peer - rapidjson::Value peer_addr; - std::string addr = replica_info.host + ":" + std::to_string(replica_info.brpc_port); - peer_addr.SetString(addr.c_str(), cast_set(addr.length()), status.GetAllocator()); - status.AddMember("remote peer", peer_addr, status.GetAllocator()); - // last failure status - rapidjson::Value compaction_status; - compaction_status.SetString( - _last_single_compaction_failure_status.c_str(), - cast_set(_last_single_compaction_failure_status.length()), - status.GetAllocator()); - status.AddMember("last failure status", compaction_status, status.GetAllocator()); - // last fetched rowset - rapidjson::Value version; - std::string fetched_version = _last_fetched_version.to_string(); - version.SetString(fetched_version.c_str(), cast_set(fetched_version.length()), - status.GetAllocator()); - status.AddMember("last fetched rowset", version, status.GetAllocator()); - root.AddMember("single replica compaction status", status, root.GetAllocator()); - } - // print all rowsets' version as an array rapidjson::Document versions_arr; rapidjson::Document missing_versions_arr; @@ -1834,33 +1806,6 @@ Status Tablet::prepare_compaction_and_calculate_permits( return Status::OK(); } -void Tablet::execute_single_replica_compaction(SingleReplicaCompaction& compaction) { - Status res = compaction.execute_compact(); - if (!res.ok()) { - set_last_failure_time(this, compaction, UnixMillis()); - set_last_single_compaction_failure_status(res.to_string()); - if (res.is()) { - DorisMetrics::instance()->single_compaction_request_cancelled->increment(1); - // "CANCELLED" indicates that the peer has not performed compaction, - // wait for the peer to perform compaction - set_skip_compaction(true, compaction.real_compact_type(), UnixSeconds()); - VLOG_CRITICAL << "Cannel fetching from the remote peer. res=" << res - << ", tablet=" << tablet_id(); - } else { - DorisMetrics::instance()->single_compaction_request_failed->increment(1); - LOG(WARNING) << "failed to do single replica compaction. res=" << res - << ", tablet=" << tablet_id(); - } - return; - } - set_last_failure_time(this, compaction, 0); -} - -bool Tablet::should_fetch_from_peer() { - return tablet_meta()->tablet_schema()->enable_single_replica_compaction() && - _engine.should_fetch_from_peer(tablet_id()); -} - std::vector Tablet::get_all_local_versions() { std::vector local_versions; { diff --git a/be/src/storage/tablet/tablet.h b/be/src/storage/tablet/tablet.h index 908d34bbf1bfb0..cc46e6f9486067 100644 --- a/be/src/storage/tablet/tablet.h +++ b/be/src/storage/tablet/tablet.h @@ -63,7 +63,6 @@ namespace doris { class Tablet; class CumulativeCompactionPolicy; class CompactionMixin; -class SingleReplicaCompaction; class RowsetWriter; struct RowsetWriterContext; class TTabletInfo; @@ -269,12 +268,6 @@ class Tablet final : public BaseTablet { _last_full_compaction_schedule_millis = millis; } - void set_last_single_compaction_failure_status(std::string status) { - _last_single_compaction_failure_status = std::move(status); - } - - void set_last_fetched_version(Version version) { _last_fetched_version = std::move(version); } - void delete_all_files(); void check_tablet_path_exists(); @@ -314,7 +307,6 @@ class Tablet final : public BaseTablet { std::shared_ptr& compaction, int64_t& permits); void execute_compaction(CompactionMixin& compaction); - void execute_single_replica_compaction(SingleReplicaCompaction& compaction); void set_cumulative_compaction_policy( std::shared_ptr cumulative_compaction_policy) { @@ -349,8 +341,6 @@ class Tablet final : public BaseTablet { _visible_version.store(visible_version); } - bool should_fetch_from_peer(); - inline bool all_beta() const { std::shared_lock rdlock(_meta_lock); return _tablet_meta->all_beta(); @@ -599,10 +589,6 @@ class Tablet final : public BaseTablet { std::string _last_base_compaction_status; std::string _last_full_compaction_status; - // single replica compaction status - std::string _last_single_compaction_failure_status; - Version _last_fetched_version; - // cumulative compaction policy std::shared_ptr _cumulative_compaction_policy; std::string_view _cumulative_compaction_type; diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index 63abb73adf699e..f637969e2b747d 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -737,10 +737,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; uint32_t highest_score = 0; - // find the single compaction tablet - uint32_t single_compact_highest_score = 0; TabletSharedPtr best_tablet; - TabletSharedPtr best_single_compact_tablet; int64_t compaction_num_per_round = ExecEnv::GetInstance()->storage_engine().to_local().get_compaction_num_per_round(); auto cmp = [](TabletScore left, TabletScore right) { return left.score > right.score; }; @@ -805,18 +802,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( return; } - // tablet should do single compaction - if (current_compaction_score > single_compact_highest_score && - tablet_ptr->should_fetch_from_peer()) { - bool ret = tablet_ptr->suitable_for_compaction(compaction_type, - cumulative_compaction_policy); - if (ret) { - single_compact_highest_score = current_compaction_score; - best_single_compact_tablet = tablet_ptr; - } - } - - if (compaction_num_per_round > 1 && !tablet_ptr->should_fetch_from_peer()) { + if (compaction_num_per_round > 1) { TabletScore ts; ts.score = current_compaction_score; ts.tablet_ptr = tablet_ptr; @@ -834,7 +820,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( } } } else { - if (current_compaction_score > highest_score && !tablet_ptr->should_fetch_from_peer()) { + if (current_compaction_score > highest_score) { bool ret = tablet_ptr->suitable_for_compaction(compaction_type, cumulative_compaction_policy); if (ret) { @@ -851,8 +837,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( VLOG_CRITICAL << "Found the best tablet for compaction. " << "compaction_type=" << compaction_type_str << ", tablet_id=" << best_tablet->tablet_id() << ", path=" << data_dir->path() - << ", highest_score=" << highest_score - << ", fetch from peer: " << best_tablet->should_fetch_from_peer(); + << ", highest_score=" << highest_score; picked_tablet.emplace_back(std::move(best_tablet)); } @@ -866,18 +851,7 @@ std::vector TabletManager::find_best_tablets_to_compaction( picked_tablet.emplace_back(*it); } - // pick single compaction tablet needs the highest score - if (best_single_compact_tablet != nullptr && single_compact_highest_score >= highest_score) { - VLOG_CRITICAL << "Found the best tablet for single compaction. " - << "compaction_type=" << compaction_type_str - << ", tablet_id=" << best_single_compact_tablet->tablet_id() - << ", path=" << data_dir->path() - << ", highest_score=" << single_compact_highest_score << ", fetch from peer: " - << best_single_compact_tablet->should_fetch_from_peer(); - picked_tablet.emplace_back(std::move(best_single_compact_tablet)); - } - *score = highest_score > single_compact_highest_score ? highest_score - : single_compact_highest_score; + *score = highest_score; return picked_tablet; } diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 4bacda4ad40b59..2058674c9e7804 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -359,11 +359,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id schema->set_enable_variant_flatten_nested(tablet_schema.variant_enable_flatten_nested); } - if (tablet_schema.__isset.enable_single_replica_compaction) { - schema->set_enable_single_replica_compaction( - tablet_schema.enable_single_replica_compaction); - } - if (tablet_schema.__isset.delete_sign_idx) { schema->set_delete_sign_idx(tablet_schema.delete_sign_idx); } diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 59b5992c6bad52..1bfc89ace185db 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1177,7 +1177,6 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac } _is_in_memory = schema.is_in_memory(); _disable_auto_compaction = schema.disable_auto_compaction(); - _enable_single_replica_compaction = schema.enable_single_replica_compaction(); _store_row_column = schema.store_row_column(); _skip_write_index_on_load = schema.skip_write_index_on_load(); _delete_sign_idx = schema.delete_sign_idx(); @@ -1274,7 +1273,6 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _next_column_unique_id = ori_tablet_schema.next_column_unique_id(); _is_in_memory = ori_tablet_schema.is_in_memory(); _disable_auto_compaction = ori_tablet_schema.disable_auto_compaction(); - _enable_single_replica_compaction = ori_tablet_schema.enable_single_replica_compaction(); _skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load(); _sort_type = ori_tablet_schema.sort_type(); _sort_col_num = ori_tablet_schema.sort_col_num(); @@ -1465,7 +1463,6 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_next_column_unique_id(cast_set(_next_column_unique_id)); tablet_schema_pb->set_is_in_memory(_is_in_memory); tablet_schema_pb->set_disable_auto_compaction(_disable_auto_compaction); - tablet_schema_pb->set_enable_single_replica_compaction(_enable_single_replica_compaction); tablet_schema_pb->set_store_row_column(_store_row_column); tablet_schema_pb->set_skip_write_index_on_load(_skip_write_index_on_load); tablet_schema_pb->set_delete_sign_idx(_delete_sign_idx); @@ -1861,7 +1858,6 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { if (a._is_in_memory != b._is_in_memory) return false; if (a._delete_sign_idx != b._delete_sign_idx) return false; if (a._disable_auto_compaction != b._disable_auto_compaction) return false; - if (a._enable_single_replica_compaction != b._enable_single_replica_compaction) return false; if (a._store_row_column != b._store_row_column) return false; if (a._row_store_page_size != b._row_store_page_size) return false; if (a._storage_page_size != b._storage_page_size) return false; diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 63af03e8cf67ef..fb86c34741b535 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -487,10 +487,6 @@ class TabletSchema : public MetadataAdder { bool deprecated_variant_flatten_nested() const { return _deprecated_enable_variant_flatten_nested; } - void set_enable_single_replica_compaction(bool enable_single_replica_compaction) { - _enable_single_replica_compaction = enable_single_replica_compaction; - } - bool enable_single_replica_compaction() const { return _enable_single_replica_compaction; } // indicate if full row store column(all the columns encodes as row) exists bool has_row_store_for_all_columns() const { return _store_row_column && row_columns_uids().empty(); @@ -819,7 +815,6 @@ class TabletSchema : public MetadataAdder { int64_t _table_id = -1; int64_t _db_id = -1; bool _disable_auto_compaction = false; - bool _enable_single_replica_compaction = false; bool _store_row_column = false; bool _skip_write_index_on_load = false; InvertedIndexStorageFormatPB _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1; diff --git a/be/test/service/http/compaction_action_test.cpp b/be/test/service/http/compaction_action_test.cpp index 09aa6bb9d7cf3c..ec25e813a3fdcb 100644 --- a/be/test/service/http/compaction_action_test.cpp +++ b/be/test/service/http/compaction_action_test.cpp @@ -107,19 +107,6 @@ TEST_F(CompactionActionTest, RunCompactionInvalidType) { EXPECT_TRUE(st.to_string().find("not supported") != std::string::npos); } -TEST_F(CompactionActionTest, RunCompactionInvalidRemoteParam) { - auto action = _make_run_action(); - HttpRequest req(_evhttp_req); - req._params[TABLET_ID_KEY] = "12345"; - req._params[PARAM_COMPACTION_TYPE] = PARAM_COMPACTION_FULL; - req._params[PARAM_COMPACTION_REMOTE] = "invalid_value"; - - std::string json_result; - Status st = action._handle_run_compaction(&req, &json_result); - EXPECT_FALSE(st.ok()); - EXPECT_TRUE(st.to_string().find("not supported") != std::string::npos); -} - TEST_F(CompactionActionTest, RunCompactionInvalidTabletId) { auto action = _make_run_action(); HttpRequest req(_evhttp_req); diff --git a/be/test/storage/compaction/single_compaction_test.cpp b/be/test/storage/compaction/single_compaction_test.cpp deleted file mode 100644 index 209bd6a3afc014..00000000000000 --- a/be/test/storage/compaction/single_compaction_test.cpp +++ /dev/null @@ -1,147 +0,0 @@ - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include - -#include -#include -#include -#include - -#include "common/status.h" -#include "gtest/gtest_pred_impl.h" -#include "storage/compaction/single_replica_compaction.h" -#include "storage/data_dir.h" -#include "storage/olap_common.h" -#include "storage/rowset/beta_rowset.h" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_meta.h" -#include "storage/storage_engine.h" -#include "storage/tablet/tablet.h" -#include "storage/tablet/tablet_meta.h" -namespace doris { - -static StorageEngine* engine_ref = nullptr; - -class SingleCompactionTest : public ::testing::Test { -protected: - void SetUp() override { - const std::string dir_path = "ut_dir/single_compact_test"; - _engine = new StorageEngine({}); - _data_dir = new DataDir(*_engine, dir_path); - engine_ref = _engine; - } - - TabletSharedPtr create_tablet(int64_t tablet_id) { - auto tablet_meta = std::make_shared(); - tablet_meta->_tablet_id = tablet_id; - (void)tablet_meta->set_partition_id(10000); - tablet_meta->set_tablet_uid({tablet_id, 0}); - tablet_meta->set_shard_id(tablet_id % 4); - tablet_meta->_schema_hash = tablet_id; - return std::make_shared(*_engine, std::move(tablet_meta), _data_dir); - } - auto create_rowset(TabletSharedPtr tablet, int64_t start, int64_t end) { - auto rowset_meta = std::make_shared(); - Version version(start, end); - rowset_meta->set_version(version); - rowset_meta->set_tablet_id(tablet->tablet_id()); - rowset_meta->set_tablet_uid(tablet->tablet_uid()); - rowset_meta->set_rowset_id(_engine->next_rowset_id()); - return std::make_shared(tablet->tablet_schema(), std::move(rowset_meta), - tablet->tablet_path()); - } - void TearDown() override { - delete _engine; - delete _data_dir; - } - -private: - StorageEngine* _engine; - DataDir* _data_dir; -}; - -TEST_F(SingleCompactionTest, test_single) { - TabletSharedPtr tablet = create_tablet(10001); - - SingleReplicaCompaction single_compaction(*engine_ref, tablet, - CompactionType::CUMULATIVE_COMPACTION); - auto st = tablet->init(); - ASSERT_TRUE(st.ok()) << st; - // load 30 rowsets - for (int i = 1; i <= 30; ++i) { - auto rs = create_rowset(tablet, i, i); - st = tablet->add_inc_rowset(rs); - ASSERT_TRUE(st.ok()) << st; - } - - // pick input rowsets, but picking is not needed now - st = single_compaction.prepare_compact(); - ASSERT_TRUE(st.ok()) << st; - - // load 2 rowsets - for (int i = 31; i <= 32; i++) { - auto rs = create_rowset(tablet, i, i); - st = tablet->add_inc_rowset(rs); - ASSERT_TRUE(st.ok()) << st; - } - - // create peer compacted rowset - auto v1 = Version(1, 32); - auto v2 = Version(33, 38); - std::vector peer_version {v1, v2}; - Version proper_version; - bool find = single_compaction._find_rowset_to_fetch(peer_version, &proper_version); - EXPECT_EQ(find, true); - EXPECT_EQ(single_compaction._input_rowsets.size(), 32); - EXPECT_EQ(single_compaction._input_rowsets.front()->start_version(), - single_compaction._output_version.first); - EXPECT_EQ(single_compaction._input_rowsets.back()->end_version(), - single_compaction._output_version.second); -} - -TEST_F(SingleCompactionTest, test_unmatch) { - TabletSharedPtr tablet = create_tablet(10000); - - SingleReplicaCompaction single_compaction(*engine_ref, tablet, - CompactionType::CUMULATIVE_COMPACTION); - auto st = tablet->init(); - ASSERT_TRUE(st.ok()) << st; - // local rowset [4-6] - auto rs = create_rowset(tablet, 4, 6); - st = tablet->add_inc_rowset(rs); - ASSERT_TRUE(st.ok()) << st; - - // pick input rowsets, but picking is not needed now - st = single_compaction.prepare_compact(); - ASSERT_TRUE(st.ok()) << st; - - // create peer compacted rowset [3-5], [6-9] - auto v1 = Version(3, 5); - auto v2 = Version(6, 9); - std::vector peer_version {v1, v2}; - Version proper_version; - bool find = single_compaction._find_rowset_to_fetch(peer_version, &proper_version); - EXPECT_EQ(find, false); // no matched version, find = false - EXPECT_EQ(single_compaction._input_rowsets.size(), 0); - EXPECT_EQ(single_compaction._output_version.first, 0); - EXPECT_EQ(single_compaction._output_version.second, 0); -} - -} // namespace doris diff --git a/be/test/storage/tablet/tablet_mgr_test.cpp b/be/test/storage/tablet/tablet_mgr_test.cpp index 3737742f014690..e145a5db77a6d8 100644 --- a/be/test/storage/tablet/tablet_mgr_test.cpp +++ b/be/test/storage/tablet/tablet_mgr_test.cpp @@ -345,7 +345,7 @@ TEST_F(TabletMgrTest, GetRowsetId) { } TEST_F(TabletMgrTest, FindTabletWithCompact) { - auto create_tablet = [this](int64_t tablet_id, bool enable_single_compact, int rowset_size) { + auto create_tablet = [this](int64_t tablet_id, int rowset_size) { std::vector cols; TColumn col1; col1.column_type.type = TPrimitiveType::SMALLINT; @@ -375,7 +375,6 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { tablet_schema.__set_storage_type(TStorageType::COLUMN); tablet_schema.__set_columns(cols); tablet_schema.__set_sequence_col_idx(1); - tablet_schema.__set_enable_single_replica_compaction(enable_single_compact); TCreateTabletReq create_tablet_req; create_tablet_req.__set_tablet_schema(tablet_schema); create_tablet_req.__set_tablet_id(tablet_id); @@ -422,7 +421,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { // create 10 tablets for (int64_t id = 1; id <= 10; ++id) { - create_tablet(id, false, rowset_size++); + create_tablet(id, rowset_size++); } std::unordered_set cumu_set; @@ -442,32 +441,8 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { ASSERT_EQ(compact_tablets[0]->tablet_id(), 10); ASSERT_EQ(score, 14); - // create 10 tablets enable single compact - // 5 tablets do cumu compaction, 5 tablets do single compaction - // if BE_TEST is defined, tablet_id % 2 == 0 means that tablet needs to do single compact - for (int64_t id = 11; id <= 20; ++id) { - create_tablet(id, true, rowset_size++); - } - - compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, - cumulative_compaction_policies); - ASSERT_EQ(compact_tablets.size(), 2); - ASSERT_EQ(compact_tablets[0]->tablet_id(), 19); - ASSERT_EQ(compact_tablets[1]->tablet_id(), 20); - ASSERT_EQ(score, 24); - - create_tablet(21, false, rowset_size++); - - compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, - cumulative_compaction_policies); - ASSERT_EQ(compact_tablets.size(), 1); - ASSERT_EQ(compact_tablets[0]->tablet_id(), 21); - ASSERT_EQ(score, 25); - // drop all tablets - for (int64_t id = 1; id <= 21; ++id) { + for (int64_t id = 1; id <= 10; ++id) { Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false); ASSERT_TRUE(drop_st.ok()) << drop_st; } @@ -475,7 +450,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { { k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 100; ++i) { - create_tablet(10000 + i, false, i); + create_tablet(10000 + i, i); } compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( @@ -496,39 +471,10 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } } - { - k_engine->_compaction_num_per_round = 10; - for (int64_t i = 1; i <= 100; ++i) { - create_tablet(20000 + i, false, i); - } - create_tablet(20102, true, 200); - - compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, &score, - cumulative_compaction_policies); - ASSERT_EQ(compact_tablets.size(), 11); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(compact_tablets[i]->tablet_id(), 20100 - i); - ASSERT_EQ(compact_tablets[i]->calc_compaction_score(), 100 - i); - } - ASSERT_EQ(compact_tablets[10]->tablet_id(), 20102); - ASSERT_EQ(compact_tablets[10]->calc_compaction_score(), 200); - - k_engine->_compaction_num_per_round = 1; - // drop all tablets - for (int64_t id = 20001; id <= 20100; ++id) { - Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false); - ASSERT_TRUE(drop_st.ok()) << drop_st; - } - - Status drop_st = _tablet_mgr->drop_tablet(20102, 20102 * 10, false); - ASSERT_TRUE(drop_st.ok()) << drop_st; - } - { k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 5; ++i) { - create_tablet(30000 + i, false, i + 5); + create_tablet(30000 + i, i + 5); } compact_tablets = _tablet_mgr->find_best_tablets_to_compaction( diff --git a/be/test/storage/test_data/header_without_inc_rs.txt b/be/test/storage/test_data/header_without_inc_rs.txt index 802fcea1c37aa2..897ba228d836c6 100644 --- a/be/test/storage/test_data/header_without_inc_rs.txt +++ b/be/test/storage/test_data/header_without_inc_rs.txt @@ -61,7 +61,6 @@ "store_row_column": false, "is_dynamic_schema": false, "is_partial_update": false, - "enable_single_replica_compaction": false, "variant_enable_flatten_nested": false }, "rs_metas": [ diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index a62e099b1bc0bf..7a6f8b5b2e9844 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -257,7 +257,6 @@ protected void createRollupReplica() throws AlterCancelException { tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.getStoragePolicy(), tbl.disableAutoCompaction(), - tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index e791c65088c7d4..fe0609261865d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2378,7 +2378,6 @@ public void updateTableProperties(Database db, String tableName, Map timeSeriesCompactionConfig, - int enableSingleCompaction, int skipWriteIndexOnLoad, + int skipWriteIndexOnLoad, int disableAutoCompaction, int verticalCompactionNumColumnsPerGroup) throws UserException { // be id -> @@ -2631,7 +2618,7 @@ public void updatePartitionProperties(Database db, String tableName, String part countDownLatch.addMark(kv.getKey(), kv.getValue()); UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory, storagePolicyId, binlogConfig, countDownLatch, compactionPolicy, - timeSeriesCompactionConfig, enableSingleCompaction, skipWriteIndexOnLoad, + timeSeriesCompactionConfig, skipWriteIndexOnLoad, disableAutoCompaction, verticalCompactionNumColumnsPerGroup); batchTask.addTask(task); } @@ -3515,7 +3502,7 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List()); - } - tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION, - Boolean.valueOf(enableSingleReplicaCompaction).toString()); - tableProperty.buildEnableSingleReplicaCompaction(); - } - - public Boolean enableSingleReplicaCompaction() { - if (tableProperty != null) { - return tableProperty.enableSingleReplicaCompaction(); - } - return false; - } - public void setStoreRowColumn(boolean storeRowColumn) { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 58e46c2ceef112..584eb6dde247ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -96,8 +96,6 @@ public class TableProperty implements GsonPostProcessable { private boolean variantEnableFlattenNested = false; - private boolean enableSingleReplicaCompaction = false; - private int verticalCompactionNumColumnsPerGroup = 5; private boolean storeRowColumn = false; @@ -165,7 +163,6 @@ public TableProperty buildProperty(short opCode) { buildTimeSeriesCompactionFileCountThreshold(); buildTimeSeriesCompactionTimeThresholdSeconds(); buildSkipWriteIndexOnLoad(); - buildEnableSingleReplicaCompaction(); buildVerticalCompactionNumColumnsPerGroup(); buildDisableAutoCompaction(); buildTimeSeriesCompactionEmptyRowsetsThreshold(); @@ -343,16 +340,6 @@ public boolean variantEnableFlattenNested() { return variantEnableFlattenNested; } - public TableProperty buildEnableSingleReplicaCompaction() { - enableSingleReplicaCompaction = Boolean.parseBoolean( - properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION, "false")); - return this; - } - - public boolean enableSingleReplicaCompaction() { - return enableSingleReplicaCompaction; - } - public TableProperty buildVerticalCompactionNumColumnsPerGroup() { verticalCompactionNumColumnsPerGroup = Integer.parseInt( properties.getOrDefault(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP, "5")); @@ -831,7 +818,6 @@ public void gsonPostProcess() throws IOException { buildTimeSeriesCompactionFileCountThreshold(); buildTimeSeriesCompactionTimeThresholdSeconds(); buildDisableAutoCompaction(); - buildEnableSingleReplicaCompaction(); buildVerticalCompactionNumColumnsPerGroup(); buildTimeSeriesCompactionEmptyRowsetsThreshold(); buildTimeSeriesCompactionLevelThreshold(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 923e1fcbc9720f..fd789dce0720fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -162,8 +162,6 @@ public class PropertyAnalyzer { @Deprecated public static final String PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED = "deprecated_variant_enable_flatten_nested"; - public static final String PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION = "enable_single_replica_compaction"; - public static final String PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP = "vertical_compaction_num_columns_per_group"; @@ -857,26 +855,6 @@ public static Boolean analyzeVariantFlattenNested(Map properties + " must be `true` or `false`"); } - public static Boolean analyzeEnableSingleReplicaCompaction(Map properties) - throws AnalysisException { - if (properties == null || properties.isEmpty()) { - return false; - } - String value = properties.get(PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION); - // set enable single replica compaction false by default - if (null == value) { - return false; - } - properties.remove(PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION); - if (value.equalsIgnoreCase("true")) { - return true; - } else if (value.equalsIgnoreCase("false")) { - return false; - } - throw new AnalysisException(PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION - + " must be `true` or `false`"); - } - public static Boolean analyzeEnableDuplicateWithoutKeysByDefault(Map properties) throws AnalysisException { if (properties == null || properties.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index b5ca1a571f5b8b..b067f5d9428acf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1533,10 +1533,6 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, olapTable.variantEnableFlattenNested().toString()); } - if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)) { - properties.put(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION, - olapTable.enableSingleReplicaCompaction().toString()); - } if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN)) { properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, olapTable.storeRowColumn().toString()); @@ -2140,7 +2136,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa indexes, tbl.isInMemory(), tabletType, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), - tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), + tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -2753,18 +2749,6 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th } olapTable.setEnableMowLightDelete(enableDeleteOnDeletePredicate); - boolean enableSingleReplicaCompaction = false; - try { - enableSingleReplicaCompaction = PropertyAnalyzer.analyzeEnableSingleReplicaCompaction(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - if (enableUniqueKeyMergeOnWrite && enableSingleReplicaCompaction) { - throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION - + " property is not supported for merge-on-write table"); - } - olapTable.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); - if (Config.isCloudMode() && ((CloudEnv) env).getEnableStorageVault()) { // Pair storageVaultInfoPair = PropertyAnalyzer.analyzeStorageVault(properties, db); diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 442c001ff2c4e0..4ddbced69a97e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -1074,7 +1074,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.getCompressionType(), olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getStoragePolicy(), olapTable.disableAutoCompaction(), - olapTable.enableSingleReplicaCompaction(), olapTable.skipWriteIndexOnLoad(), olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), olapTable.getTimeSeriesCompactionFileCountThreshold(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java index 27973552d4a9c6..40179c8fdbf3d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java @@ -286,17 +286,6 @@ public void validate(ConnectContext ctx) throws UserException { throw new AnalysisException( "Property " + PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT + " is not allowed to change"); - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)) { - if (!properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION).equalsIgnoreCase("true") - && !properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION) - .equalsIgnoreCase("false")) { - throw new AnalysisException( - "Property " - + PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION - + " should be set to true or false"); - } - this.needTableStable = false; - this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)) { if (!properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE) .equalsIgnoreCase("true") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 6f34088deb1b73..ba2580f4408e87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -253,7 +253,7 @@ public class OperationType { // The actual opcode now represents // modify table properties: inMemory, StoragePolicy, IsBeingSynced, CompactionPolicy, // TimeSeriesCompactionFileCountThreshold, SeriesCompactionTimeThresholdSeconds, - // SkipWriteIndexOnLoad, EnableSingleReplicaCompaction. + // SkipWriteIndexOnLoad. public static final short OP_MODIFY_TABLE_PROPERTIES = 267; // set table default distribution bucket num diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 6ffae4c6660979..add182b117c070 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -108,8 +108,6 @@ public class CreateReplicaTask extends AgentTask { private boolean disableAutoCompaction; - private boolean enableSingleReplicaCompaction; - private boolean skipWriteIndexOnLoad; private String compactionPolicy; @@ -150,7 +148,6 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition TCompressionType compressionType, boolean enableUniqueKeyMergeOnWrite, String storagePolicy, boolean disableAutoCompaction, - boolean enableSingleReplicaCompaction, boolean skipWriteIndexOnLoad, String compactionPolicy, long timeSeriesCompactionGoalSizeMbytes, @@ -201,7 +198,6 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition } } this.disableAutoCompaction = disableAutoCompaction; - this.enableSingleReplicaCompaction = enableSingleReplicaCompaction; this.skipWriteIndexOnLoad = skipWriteIndexOnLoad; this.compactionPolicy = compactionPolicy; this.timeSeriesCompactionGoalSizeMbytes = timeSeriesCompactionGoalSizeMbytes; @@ -374,7 +370,6 @@ public TCreateTabletReq toThrift() { tSchema.setIsInMemory(isInMemory); tSchema.setDisableAutoCompaction(disableAutoCompaction); tSchema.setVariantEnableFlattenNested(variantEnableFlattenNested); - tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction); tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); tSchema.setStoreRowColumn(storeRowColumn); tSchema.setRowStorePageSize(rowStorePageSize); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java index 4ba127f447286e..7977df03456fe3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java @@ -51,7 +51,6 @@ public class UpdateTabletMetaInfoTask extends AgentTask { // For ReportHandler private List tabletMetaInfos; // < 0 means not to update property, > 0 means true, == 0 means false - private int enableSingleReplicaCompaction = -1; private int skipWriteIndexOnLoad = -1; private int disableAutoCompaction = -1; private int verticalCompactionNumColumnsPerGroup = -1; @@ -88,14 +87,12 @@ public UpdateTabletMetaInfoTask(long backendId, MarkedCountDownLatch>> latch, String compactionPolicy, Map timeSeriesCompactionConfig, - int enableSingleReplicaCompaction, int skipWriteIndexOnLoad, int disableAutoCompaction, int verticalCompactionNumColumnsPerGroup) { this(backendId, tableIdWithSchemaHash, inMemory, storagePolicyId, binlogConfig, latch); this.compactionPolicy = compactionPolicy; this.timeSeriesCompactionConfig = timeSeriesCompactionConfig; - this.enableSingleReplicaCompaction = enableSingleReplicaCompaction; this.skipWriteIndexOnLoad = skipWriteIndexOnLoad; this.disableAutoCompaction = disableAutoCompaction; this.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup; @@ -173,9 +170,6 @@ public TUpdateTabletMetaInfoReq toThrift() { .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_LEVEL_THRESHOLD)); } } - if (enableSingleReplicaCompaction >= 0) { - metaInfo.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction > 0); - } if (skipWriteIndexOnLoad >= 0) { metaInfo.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad > 0); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTestData.json b/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTestData.json index c64451fd37d9e6..ed28fed7998bef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTestData.json +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTestData.json @@ -456,7 +456,6 @@ "default.replication_allocation": "tag.location.default: 1", "storage_format": "V2", "storage_policy": "", - "enable_single_replica_compaction": "false", "compression": "LZ4F", "store_row_column": "false", "light_schema_change": "true", diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 1dfbca127dee47..ffc0ac714a9a99 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -412,8 +412,7 @@ public void testProject() throws Exception { + " \"is_being_synced\" = \"false\",\n" + " \"storage_format\" = \"V2\",\n" + " \"light_schema_change\" = \"true\",\n" - + " \"disable_auto_compaction\" = \"false\",\n" - + " \"enable_single_replica_compaction\" = \"false\"\n" + + " \"disable_auto_compaction\" = \"false\"\n" + " )" ); createView("create view IF NOT EXISTS test_view7_drop_nereids (k1,k2,k3,k4) as\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index 4b71c38bbd9f95..132e0404896eed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -115,7 +115,7 @@ public void setUp() throws AnalysisException { createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, - TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, + TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, 5); // drop diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 27a69ce4bcfb4b..be6715333ccb9d 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -472,7 +472,7 @@ message TabletSchemaPB { optional bool is_dynamic_schema = 19 [default=false]; // deprecated reserved 20; // deprecated is_partial_update reserved 21; // deprecated partial_update_input_columns - optional bool enable_single_replica_compaction = 22 [default=false]; + reserved 22; // deprecated enable_single_replica_compaction optional bool skip_write_index_on_load = 23 [default=false]; repeated int32 cluster_key_uids = 24; optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; @@ -514,7 +514,7 @@ message TabletSchemaCloudPB { optional bool store_row_column = 18 [default=false]; // store tuplerow oriented column reserved 20; // deprecated is_partial_update reserved 21; // deprecated partial_update_input_columns - optional bool enable_single_replica_compaction = 22 [default=false]; + reserved 22; // deprecated enable_single_replica_compaction optional bool skip_write_index_on_load = 23 [default=false]; repeated int32 cluster_key_uids = 24; optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 8041b8484ac18b..1d3802c974c049 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -42,7 +42,7 @@ struct TTabletSchema { 14: optional i32 version_col_idx = -1 15: optional bool is_dynamic_schema = false // deprecated 16: optional bool store_row_column = false - 17: optional bool enable_single_replica_compaction = false + // 17: deprecated enable_single_replica_compaction 18: optional bool skip_write_index_on_load = false 19: optional list cluster_key_uids // col unique id for row store column @@ -540,7 +540,7 @@ struct TTabletMetaInfo { 11: optional i64 time_series_compaction_goal_size_mbytes 12: optional i64 time_series_compaction_file_count_threshold 13: optional i64 time_series_compaction_time_threshold_seconds - 14: optional bool enable_single_replica_compaction + // 14: deprecated enable_single_replica_compaction 15: optional bool skip_write_index_on_load 16: optional bool disable_auto_compaction 17: optional i64 time_series_compaction_empty_rowsets_threshold diff --git a/regression-test/data/cloud_p0/cache/ttl/ex06_ttl_restart_consistency.out b/regression-test/data/cloud_p0/cache/ttl/ex06_ttl_restart_consistency.out index 4ca2398c918dd5..c9f49789b718ee 100644 --- a/regression-test/data/cloud_p0/cache/ttl/ex06_ttl_restart_consistency.out +++ b/regression-test/data/cloud_p0/cache/ttl/ex06_ttl_restart_consistency.out @@ -3,10 +3,10 @@ 300 -- !ex06_create_table_1 -- -ex06_ttl_restart_consistency CREATE TABLE `ex06_ttl_restart_consistency` (\n `k1` bigint NOT NULL,\n `c1` varchar(64) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 8\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +ex06_ttl_restart_consistency CREATE TABLE `ex06_ttl_restart_consistency` (\n `k1` bigint NOT NULL,\n `c1` varchar(64) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 8\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !ex06_create_table_2 -- -ex06_ttl_restart_consistency CREATE TABLE `ex06_ttl_restart_consistency` (\n `k1` bigint NOT NULL,\n `c1` varchar(64) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 8\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +ex06_ttl_restart_consistency CREATE TABLE `ex06_ttl_restart_consistency` (\n `k1` bigint NOT NULL,\n `c1` varchar(64) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 8\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !ex06_after_restart -- 300 \ No newline at end of file diff --git a/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out b/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out index 543d7e19b73862..1d2ad674a77720 100644 --- a/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out +++ b/regression-test/data/cloud_p0/cache/ttl/test_show_ttl.out @@ -6,11 +6,11 @@ 0 -- !test_show_ttl_3 -- -customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "300",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"compression" = "ZSTD",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "300",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"compression" = "ZSTD",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !test_show_ttl_4 -- 0 -- !test_show_ttl_5 -- -customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"compression" = "ZSTD",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +customer_ttl CREATE TABLE `customer_ttl` (\n `C_CUSTKEY` int NOT NULL,\n `C_NAME` varchar(25) NOT NULL,\n `C_ADDRESS` varchar(40) NOT NULL,\n `C_NATIONKEY` int NOT NULL,\n `C_PHONE` char(15) NOT NULL,\n `C_ACCTBAL` decimal(15,2) NOT NULL,\n `C_MKTSEGMENT` char(10) NOT NULL,\n `C_COMMENT` varchar(117) NOT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`C_CUSTKEY`, `C_NAME`)\nDISTRIBUTED BY HASH(`C_CUSTKEY`) BUCKETS 32\nPROPERTIES (\n"file_cache_ttl_seconds" = "0",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"compression" = "ZSTD",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/data/compaction/test_single_compaction_fault_injection.out b/regression-test/data/compaction/test_single_compaction_fault_injection.out deleted file mode 100644 index 1237b8f976bb51..00000000000000 --- a/regression-test/data/compaction/test_single_compaction_fault_injection.out +++ /dev/null @@ -1,13 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -1 a 100 -1 b 100 -2 a 100 -2 b 100 -3 a 100 -3 b 100 -5 a 100 -6 a 100 -7 a 100 -8 a 100 - diff --git a/regression-test/data/compaction/test_single_compaction_with_variant_inverted_index.out b/regression-test/data/compaction/test_single_compaction_with_variant_inverted_index.out deleted file mode 100644 index 045b7675a915e2..00000000000000 --- a/regression-test/data/compaction/test_single_compaction_with_variant_inverted_index.out +++ /dev/null @@ -1,4 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -6 - diff --git a/regression-test/data/compaction/test_single_replica_compaction.out b/regression-test/data/compaction/test_single_replica_compaction.out deleted file mode 100644 index 9895dbe9e11f0a..00000000000000 --- a/regression-test/data/compaction/test_single_replica_compaction.out +++ /dev/null @@ -1,10 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -1 b 100 -2 b 100 -3 b 100 -5 a 100 -6 a 100 -7 a 100 -8 a 100 - diff --git a/regression-test/data/insert_p0/test_group_commit_interval_ms_property.out b/regression-test/data/insert_p0/test_group_commit_interval_ms_property.out index 4b2a17ea07a0d5..83e2ecaf300497 100644 --- a/regression-test/data/insert_p0/test_group_commit_interval_ms_property.out +++ b/regression-test/data/insert_p0/test_group_commit_interval_ms_property.out @@ -1,13 +1,13 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !1 -- -test_group_commit_interval_ms_property_tbl_legacy CREATE TABLE `test_group_commit_interval_ms_property_tbl_legacy` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000"\n); +test_group_commit_interval_ms_property_tbl_legacy CREATE TABLE `test_group_commit_interval_ms_property_tbl_legacy` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000"\n); -- !2 -- -test_group_commit_interval_ms_property_tbl_legacy CREATE TABLE `test_group_commit_interval_ms_property_tbl_legacy` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "1000"\n); +test_group_commit_interval_ms_property_tbl_legacy CREATE TABLE `test_group_commit_interval_ms_property_tbl_legacy` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "1000"\n); -- !1 -- -test_group_commit_interval_ms_property_tbl_nereids CREATE TABLE `test_group_commit_interval_ms_property_tbl_nereids` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000"\n); +test_group_commit_interval_ms_property_tbl_nereids CREATE TABLE `test_group_commit_interval_ms_property_tbl_nereids` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000"\n); -- !2 -- -test_group_commit_interval_ms_property_tbl_nereids CREATE TABLE `test_group_commit_interval_ms_property_tbl_nereids` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "1000"\n); +test_group_commit_interval_ms_property_tbl_nereids CREATE TABLE `test_group_commit_interval_ms_property_tbl_nereids` (\n `k` BIGINT NULL,\n `v` BIGINT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`v`) BUCKETS 8\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "false",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "1000"\n); diff --git a/regression-test/data/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.out b/regression-test/data/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.out deleted file mode 100644 index d00ff307840b20..00000000000000 --- a/regression-test/data/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.out +++ /dev/null @@ -1,9 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_default -- -1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 - --- !select_default2 -- -1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 - diff --git a/regression-test/data/query_p0/show/test_show_create_table.out b/regression-test/data/query_p0/show/test_show_create_table.out index 0f621de01726b4..81989ebaaf78ab 100644 --- a/regression-test/data/query_p0/show/test_show_create_table.out +++ b/regression-test/data/query_p0/show/test_show_create_table.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date MAX NOT NULL COMMENT 'e',\n `datetimev1` datetime MAX NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) MAX NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) MAX NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nAGGREGATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`datek1`) BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date MAX NOT NULL COMMENT 'e',\n `datetimev1` datetime MAX NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) MAX NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) MAX NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nAGGREGATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`datek1`) BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807"\n); -- !select -- -tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date NOT NULL COMMENT 'e',\n `datetimev1` datetime NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nDUPLICATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date NOT NULL COMMENT 'e',\n `datetimev1` datetime NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nDUPLICATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807"\n); diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index 8583440960d3cc..7d27a6a43a451b 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_check_1 -- -114 +111 -- !select_check_2 -- internal test_table_properties_db duplicate_table _auto_bucket false @@ -15,7 +15,6 @@ internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false -internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 @@ -52,7 +51,6 @@ internal test_table_properties_db listtable data_sort.sort_type LEXICAL internal test_table_properties_db listtable deprecated_variant_enable_flatten_nested false internal test_table_properties_db listtable disable_auto_compaction false internal test_table_properties_db listtable enable_mow_light_delete false -internal test_table_properties_db listtable enable_single_replica_compaction false internal test_table_properties_db listtable enable_unique_key_merge_on_write false internal test_table_properties_db listtable file_cache_ttl_seconds 0 internal test_table_properties_db listtable group_commit_data_bytes 134217728 @@ -89,7 +87,6 @@ internal test_table_properties_db unique_table data_sort.sort_type LEXICAL internal test_table_properties_db unique_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db unique_table disable_auto_compaction false internal test_table_properties_db unique_table enable_mow_light_delete false -internal test_table_properties_db unique_table enable_single_replica_compaction false internal test_table_properties_db unique_table enable_unique_key_merge_on_write true internal test_table_properties_db unique_table file_cache_ttl_seconds 0 internal test_table_properties_db unique_table group_commit_data_bytes 134217728 @@ -128,7 +125,6 @@ internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false -internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 @@ -165,7 +161,6 @@ internal test_table_properties_db unique_table data_sort.sort_type LEXICAL internal test_table_properties_db unique_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db unique_table disable_auto_compaction false internal test_table_properties_db unique_table enable_mow_light_delete false -internal test_table_properties_db unique_table enable_single_replica_compaction false internal test_table_properties_db unique_table enable_unique_key_merge_on_write true internal test_table_properties_db unique_table file_cache_ttl_seconds 0 internal test_table_properties_db unique_table group_commit_data_bytes 134217728 @@ -206,7 +201,6 @@ internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false -internal test_table_properties_db duplicate_table enable_single_replica_compaction false internal test_table_properties_db duplicate_table enable_unique_key_merge_on_write false internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out index 7929b470514276..e0600604f51f35 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_nereids_view CREATE VIEW `show_create_table_and_view 300 1 -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/data/show_p0/test_show_table_and_views.out b/regression-test/data/show_p0/test_show_table_and_views.out index 67b8d4e06109d2..1b7f4e016c9d04 100644 --- a/regression-test/data/show_p0/test_show_table_and_views.out +++ b/regression-test/data/show_p0/test_show_table_and_views.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_table_and_views_table CREATE TABLE `show_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false"\n); +show_table_and_views_table CREATE TABLE `show_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false"\n); -- !select -- 1 300 2 @@ -36,5 +36,5 @@ show_table_and_views_view CREATE VIEW `show_table_and_views_view` COMMENT 'VIEW' 300 1 -- !show -- -show_table_and_views_table CREATE TABLE `show_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false"\n); +show_table_and_views_table CREATE TABLE `show_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false"\n); diff --git a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy deleted file mode 100644 index 5220d07914347c..00000000000000 --- a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy +++ /dev/null @@ -1,342 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_single_compaction_fault_injection", "p2, nonConcurrent") { - if (isCloudMode()) { - logger.info("Skip test in cloud mode") - return - } - def tableName = "test_single_compaction" - - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - if (compact_type == "cumulative") { - def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) - assertEquals(code_1, 0) - return out_1 - } else if (compact_type == "full") { - def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) - assertEquals(code_2, 0) - return out_2 - } else { - assertFalse(True) - } - } - - def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative&remote=true") - - Integer maxRetries = 10; // Maximum number of retries - Integer retryCount = 0; // Current retry count - Integer sleepTime = 5000; // Sleep time in milliseconds - String cmd = sb.toString() - def process - int code_3 - String err_3 - String out_3 - - while (retryCount < maxRetries) { - process = cmd.execute() - code_3 = process.waitFor() - err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))) - out_3 = process.getText() - - // If the command was successful, break the loop - if (code_3 == 0) { - break - } - - // If the command was not successful, increment the retry count, sleep for a while and try again - retryCount++ - sleep(sleepTime) - } - assertEquals(code_3, 0) - logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3) - return out_3 - } - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/show?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get tablet status: =" + code + ", out=" + out) - assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } - - // find the master be for single compaction - Boolean found = false - String master_backend_id - List follower_backend_id = new ArrayList<>() - String tablet_id - def tablets - try { - GetDebugPoint().enableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_num" = "2", "enable_single_replica_compaction" = "true", "compaction_policy" = "time_series"); - """ - - tablets = sql_return_maparray """ show tablets from ${tableName}; """ - // wait for update replica infos - Thread.sleep(70000) - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - found = false - logger.warn("multipe master"); - break; - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - assertFalse(found) - assertFalse(master_backend_id.isEmpty()) - assertTrue(follower_backend_id.isEmpty()) - master_backend_id = "" - } finally { - GetDebugPoint().disableDebugPointForAllFEs('getTabletReplicaInfos.returnEmpty') - // wait for update replica infos - Thread.sleep(70000) - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master") - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - assertTrue(found) - assertFalse(master_backend_id.isEmpty()) - assertFalse(follower_backend_id.isEmpty()) - } - - - def checkSucceedCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) - } - } - - def checkFailedCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertFalse(master_rowsets.size() == rowsets.size()) - } - } - - // return ok - try { - GetDebugPoint().enableDebugPointForAllBEs("do_single_compaction_return_ok"); - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id); - } - } finally { - GetDebugPoint().disableDebugPointForAllBEs("do_single_compaction_return_ok"); - } - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ - - // trigger master be to do compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer"); - for (String id in follower_backend_id) { - def out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet don't have peer replica")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer") - } - - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_get_peer_versions"); - for (String id in follower_backend_id) { - def out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("tablet failed get peer versions")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_get_peer_versions") - } - - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_make_snapshot"); - for (String id in follower_backend_id) { - def out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed snapshot")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_make_snapshot") - } - - try { - GetDebugPoint().enableDebugPointForAllBEs("single_compaction_failed_download_file"); - for (String id in follower_backend_id) { - def out = triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - assertTrue(out.contains("compaction task is successfully triggered") || out.contains("failed to download file")); - } - checkFailedCompactionResult.call() - } finally { - GetDebugPoint().disableDebugPointForAllBEs("single_compaction_failed_download_file") - } - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } - - // check rowsets - checkSucceedCompactionResult.call() - - sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ - sql """ DELETE FROM ${tableName} WHERE id = 4; """ - sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ - - // trigger master be to do compaction with delete - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } - - // check rowsets - checkSucceedCompactionResult.call() - - qt_sql """ - select * from ${tableName} order by id, name, score - """ -} \ No newline at end of file diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy deleted file mode 100644 index 51b99b8878c26d..00000000000000 --- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy +++ /dev/null @@ -1,257 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_single_compaction_with_variant_inverted", "p2, nonConcurrent") { - if (isCloudMode()) { - return; - } - def tableName = "test_single_compaction_with_variant_inverted" - - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - - boolean disableAutoCompaction = true - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - } - } - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=${compact_type}") - - String command = sb.toString() - logger.info(command) - def process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success, " + out2 - } - assertEquals(code2, 0) - return out2 - } - - def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative&remote=true") - - String command = sb.toString() - logger.info(command) - def process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() - logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err) - if (!disableAutoCompaction) { - return "Success, " + out3 - } - assertEquals(code3, 0) - return out3 - } - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - def process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/show?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - def process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get tablet status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } - - - sql """ DROP TABLE IF EXISTS ${tableName}; """ - setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) { - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL, - `properties` variant, - INDEX idx_props (`properties`) USING INVERTED PROPERTIES("parser" = "none") COMMENT '' - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "2", - "enable_single_replica_compaction" = "true", - "inverted_index_storage_format" = "V1", - "compaction_policy" = "time_series" - ); - """ - } - - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // wait for update replica infos - Thread.sleep(72000) - - // find the master be for single replica compaction - Boolean found = false - String master_backend_id; - List follower_backend_id = new ArrayList<>() - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - String tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master"); - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - - def checkCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) - } - } - - def checkTabletFileCrc = { - def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) - - for (String backend: follower_backend_id) { - def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) - assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) - assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) - assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) - assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) - assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) - } - } - - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100, '{"%a" : 1234, "@point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100, '{"@a" : 1234, "%point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100, '{"%a" : 1234, "%point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100, '{"@a" : 1234, "@point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100, '{"a" : 1234, "point" : 1, "xxxx" : "ddddd"}'); """ - - - // trigger master be to do full compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } - - // check rowsets - checkCompactionResult.call() - checkTabletFileCrc.call() - - qt_sql """ - select count() from ${tableName} where cast(properties['xxxx'] as string) MATCH_ANY 'ddddd'; - """ - - sql """ DROP TABLE IF EXISTS ${tableName}; """ -} diff --git a/regression-test/suites/compaction/test_single_replica_compaction.groovy b/regression-test/suites/compaction/test_single_replica_compaction.groovy deleted file mode 100644 index 53f241dac31788..00000000000000 --- a/regression-test/suites/compaction/test_single_replica_compaction.groovy +++ /dev/null @@ -1,264 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_single_compaction_p2", "p2") { - if (isCloudMode()) { - return; - } - def tableName = "test_single_replica_compaction" - - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - if (compact_type == "cumulative") { - def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) - assertEquals(code_1, 0) - return out_1 - } else if (compact_type == "full") { - def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) - assertEquals(code_2, 0) - return out_2 - } else { - assertFalse(True) - } - } - - def triggerSingleCompaction = { be_host, be_http_port, tablet_id -> - StringBuilder sb = new StringBuilder(); - sb.append("curl -X POST http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run?tablet_id=") - sb.append(tablet_id) - sb.append("&compact_type=cumulative&remote=true") - - Integer maxRetries = 10; // Maximum number of retries - Integer retryCount = 0; // Current retry count - Integer sleepTime = 5000; // Sleep time in milliseconds - String cmd = sb.toString() - def process - int code_3 - String err_3 - String out_3 - - while (retryCount < maxRetries) { - process = cmd.execute() - code_3 = process.waitFor() - err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))) - out_3 = process.getText() - - // If the command was successful, break the loop - if (code_3 == 0) { - break - } - - // If the command was not successful, increment the retry count, sleep for a while and try again - retryCount++ - sleep(sleepTime) - } - assertEquals(code_3, 0) - logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3) - return out_3 - } - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - sb.append("curl -X GET ${protocol}://${be_host}:${be_http_port}") - sb.append("/api/compaction/show?tablet_id=") - sb.append(tablet_id) - if (enableTls) { - sb.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - sb.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - sb.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - - String command = sb.toString() - logger.info(command) - def process = command.execute() - code = process.waitFor() - out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } - - - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(255) NULL, - `score` int(11) NULL - ) ENGINE=OLAP - UNIQUE KEY(`id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "2", - "enable_single_replica_compaction" = "true", - "enable_unique_key_merge_on_write" = "false" - ); - """ - - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // wait for update replica infos - Thread.sleep(70000) - - // find the master be for single replica compaction - Boolean found = false - String master_backend_id; - List follower_backend_id = new ArrayList<>() - // The test table only has one bucket with 2 replicas, - // and `show tablets` will return 2 different replicas with the same tablet. - // So we can use the same tablet_id to get tablet/trigger compaction with different backends. - String tablet_id = tablets[0].TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - for (def tablet in tablets) { - String trigger_backend_id = tablet.BackendId - def tablet_status = getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - if (!tablet_status.containsKey("single replica compaction status")) { - if (found) { - logger.warn("multipe master"); - assertTrue(false) - } - found = true - master_backend_id = trigger_backend_id - } else { - follower_backend_id.add(trigger_backend_id) - } - } - - def checkCompactionResult = { - def master_tablet_status = getTabletStatus(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id); - def master_rowsets = master_tablet_status."rowsets" - assert master_rowsets instanceof List - logger.info("rowset size: " + master_rowsets.size()) - - for (String backend: follower_backend_id) { - def tablet_status = getTabletStatus(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id); - def rowsets = tablet_status."rowsets" - assert rowsets instanceof List - assertEquals(master_rowsets.size(), rowsets.size()) - } - } - - def checkTabletFileCrc = { - def (master_code, master_out, master_err) = calc_file_crc_on_tablet(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[master_backend_id] + " code=" + master_code + ", out=" + master_out + ", err=" + master_err) - - for (String backend: follower_backend_id) { - def (follower_code, follower_out, follower_err) = calc_file_crc_on_tablet(backendId_to_backendIP[backend], backendId_to_backendHttpPort[backend], tablet_id) - logger.info("Run calc_file_crc_on_tablet: ip=" + backendId_to_backendIP[backend] + " code=" + follower_code + ", out=" + follower_out + ", err=" + follower_err) - assertTrue(parseJson(follower_out.trim()).crc_value == parseJson(master_out.trim()).crc_value) - assertTrue(parseJson(follower_out.trim()).start_version == parseJson(master_out.trim()).start_version) - assertTrue(parseJson(follower_out.trim()).end_version == parseJson(master_out.trim()).end_version) - assertTrue(parseJson(follower_out.trim()).file_count == parseJson(master_out.trim()).file_count) - assertTrue(parseJson(follower_out.trim()).rowset_count == parseJson(master_out.trim()).rowset_count) - } - } - - sql """ INSERT INTO ${tableName} VALUES (1, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (1, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (2, "b", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (3, "b", 100); """ - - // trigger master be to do compaction - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } - - // check rowsets - checkCompactionResult.call() - - sql """ INSERT INTO ${tableName} VALUES (4, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (5, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (6, "a", 100); """ - sql """ DELETE FROM ${tableName} WHERE id = 4; """ - sql """ INSERT INTO ${tableName} VALUES (7, "a", 100); """ - sql """ INSERT INTO ${tableName} VALUES (8, "a", 100); """ - - // trigger master be to do compaction with delete - assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], - "full", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id) - - // trigger follower be to fetch compaction result - for (String id in follower_backend_id) { - assertTrue(triggerSingleCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[id], backendId_to_backendHttpPort[id], tablet_id) - } - - // check rowsets - checkCompactionResult.call() - checkTabletFileCrc.call() - - qt_sql """ - select * from ${tableName} order by id - """ - -} diff --git a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy index 56532fc2b07c12..5edff4e38e0da5 100644 --- a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy +++ b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy @@ -257,14 +257,6 @@ suite("test_table_level_compaction_policy") { ); """ sql """sync""" - if (!isCloudMode()) { - test { - sql """ - alter table ${tableName} set ("enable_single_replica_compaction" = "true") - """ - exception "enable_single_replica_compaction property is not supported for merge-on-write table" - } - } sql """ DROP TABLE IF EXISTS ${tableName} """ sql """sync""" } diff --git a/regression-test/suites/correctness_p0/test_and_or.groovy b/regression-test/suites/correctness_p0/test_and_or.groovy index 8ad00122baff27..001c69a2dbc4ad 100644 --- a/regression-test/suites/correctness_p0/test_and_or.groovy +++ b/regression-test/suites/correctness_p0/test_and_or.groovy @@ -43,8 +43,7 @@ suite("test_and_or") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/correctness_p0/test_cast_null.groovy b/regression-test/suites/correctness_p0/test_cast_null.groovy index cfbbf4b8034900..ac4ff99edc2520 100644 --- a/regression-test/suites/correctness_p0/test_cast_null.groovy +++ b/regression-test/suites/correctness_p0/test_cast_null.groovy @@ -35,8 +35,7 @@ suite("test_cast_null") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -51,8 +50,7 @@ suite("test_cast_null") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -84,8 +82,7 @@ suite("test_cast_null") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """insert into test_table_tabc values(1.0);""" diff --git a/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy b/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy index f6914eeb7b4ce1..588d3a84c2c8b2 100644 --- a/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy +++ b/regression-test/suites/correctness_p0/test_colocate_join_with_different_tablets.groovy @@ -299,7 +299,6 @@ suite("test_colocate_join_with_different_tablets") { "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); @@ -953,8 +952,7 @@ suite("test_colocate_join_with_different_tablets") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "binlog.enable" = "false", - "enable_single_replica_compaction" = "false" + "binlog.enable" = "false" ) """ diff --git a/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy b/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy index 660d2174169950..e7c8be242229bd 100644 --- a/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy +++ b/regression-test/suites/correctness_p0/test_ctas_mv/test_ctas_mv.groovy @@ -43,8 +43,7 @@ suite("test_ctas_mv") { "replication_allocation"="tag.location.default:1", "is_being_synced"="false", "storage_format"="V2", - "disable_auto_compaction"="false", - "enable_single_replica_compaction"="false" + "disable_auto_compaction"="false" ); """ @@ -58,8 +57,7 @@ suite("test_ctas_mv") { "replication_allocation"="tag.location.default:1", "is_being_synced"="false", "storage_format"="V2", - "disable_auto_compaction"="false", - "enable_single_replica_compaction"="false" + "disable_auto_compaction"="false" ); """ diff --git a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy index 5efa3a16b497ca..d52e8dd2acc383 100644 --- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy +++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy @@ -88,7 +88,6 @@ suite("test_mv_case") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -106,7 +105,6 @@ suite("test_mv_case") { "inverted_index_storage_format" = "V1", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_nondeterministic_function" = "true" @@ -129,7 +127,6 @@ suite("test_mv_case") { "inverted_index_storage_format" = "V1", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_nondeterministic_function" = "true" diff --git a/regression-test/suites/correctness_p0/test_nested_lateral_view.groovy b/regression-test/suites/correctness_p0/test_nested_lateral_view.groovy index 1d42331ee8808e..ce464accfd2045 100644 --- a/regression-test/suites/correctness_p0/test_nested_lateral_view.groovy +++ b/regression-test/suites/correctness_p0/test_nested_lateral_view.groovy @@ -38,8 +38,7 @@ suite("test_nested_lateral_view") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );""" sql """INSERT INTO ods_42378777c342d2da36d05db875393811 (`57CE5E02901753F7_06a28`,`57CE5E02520653F7_06a28`,`65F695F4519253F7_06a28`,`65E5671F659C6760_06a28`,`57CE5E027A7A683C_06a28`,`57CE5E02767E520653F7_06a28a4`,`65705B575C0F657070B9_06a28a4`,qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq_,`5F88591A7B2653F7_06a28`) VALUES ('a,a,a,a','a;a;a;a','1899-12-31 15:11:22','2024-01-12 00:00:00','a a a a','a%a%a%a','33..33.*.@3',3.3300000000,''), diff --git a/regression-test/suites/correctness_p0/test_set_operation.groovy b/regression-test/suites/correctness_p0/test_set_operation.groovy index 0f80729571b8bb..084c6502de18ad 100644 --- a/regression-test/suites/correctness_p0/test_set_operation.groovy +++ b/regression-test/suites/correctness_p0/test_set_operation.groovy @@ -87,8 +87,7 @@ suite("test_set_operation") { "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" ); + "disable_auto_compaction" = "false" ); """ @@ -102,8 +101,7 @@ suite("test_set_operation") { "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" ); + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy index b183e00243f05a..8e3508a7b2736a 100644 --- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy @@ -281,8 +281,7 @@ suite("test_decimalv2_overflow2") { "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ insert into test_table values ('2023-12-18', '95357.100000000000000000000000000000000000')""" qt_sql """ select * from test_table """ diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_string.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_string.groovy index 0681ae6d301c79..cc1754416c528f 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_string.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_string.groovy @@ -20,7 +20,7 @@ suite("test_decimalv3_string") { sql "CREATE DATABASE IF NOT EXISTS ${db}" sql "use ${db}" sql "drop table if exists replicationtesttable" - sql '''CREATE TABLE `replicationtesttable` ( `k1` varchar(144) NOT NULL COMMENT '_key__', `k2` int(11) NOT NULL COMMENT '_key__', v4 decimal(18,4) null, ) ENGINE=OLAP UNIQUE KEY(`k1`, `k2`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 32 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", "enable_single_replica_compaction" = "false" ); ''' + sql '''CREATE TABLE `replicationtesttable` ( `k1` varchar(144) NOT NULL COMMENT '_key__', `k2` int(11) NOT NULL COMMENT '_key__', v4 decimal(18,4) null, ) ENGINE=OLAP UNIQUE KEY(`k1`, `k2`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 32 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false" ); ''' sql "insert into replicationtesttable (k1,k2,v4) select 'aaa',10,22500000900.000000000000000000000000000000" sql "insert into replicationtesttable (k1,k2,v4) select 'bbb',20,'22500000900.000000000000000000000000000000'" diff --git a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_large_string_and_nested_type.groovy b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_large_string_and_nested_type.groovy index 28a68681f7c188..8af01a51d3b464 100644 --- a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_large_string_and_nested_type.groovy +++ b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_large_string_and_nested_type.groovy @@ -36,7 +36,6 @@ suite("test_large_string_and_nested_type", "p0") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy index b7a8572b898b2b..a199ccaf6ec7d4 100644 --- a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy +++ b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy @@ -166,7 +166,6 @@ suite("test_special_buffer_before_nested", "p0") { "store_row_column" = "true", "row_store_page_size" = "16384", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "enable_mow_light_delete" = "false" ); """ diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy index 6f870e29ce9283..15cc424c3a657e 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy @@ -42,8 +42,7 @@ suite("test_nested_type_with_resize") { "storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ streamLoad { diff --git a/regression-test/suites/ddl_p0/create_view_nereids/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/create_view_nereids/test_create_view_nereids.groovy index eea4db4cef94cc..4143ae1dd1ece5 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/test_create_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/test_create_view_nereids.groovy @@ -131,8 +131,7 @@ suite("test_create_view_nereids") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """insert into view_baseall_nereids values(1,[1,2,3]);""" diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy index 4ae75c67c94283..c1d29a95b6f91c 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy @@ -162,7 +162,6 @@ suite("test_generated_column_stream_mysql_load") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "5000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false", diff --git a/regression-test/suites/ddl_p0/test_ctas.groovy b/regression-test/suites/ddl_p0/test_ctas.groovy index df304aeb28430c..5e556bc7ac2d45 100644 --- a/regression-test/suites/ddl_p0/test_ctas.groovy +++ b/regression-test/suites/ddl_p0/test_ctas.groovy @@ -258,8 +258,7 @@ suite("test_ctas") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );''' sql '''insert into test_ctas_of_view values ('a');''' diff --git a/regression-test/suites/ddl_p0/test_drop_view_nereids.groovy b/regression-test/suites/ddl_p0/test_drop_view_nereids.groovy index 30642b100495a7..2c06ccbf9f511c 100644 --- a/regression-test/suites/ddl_p0/test_drop_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_drop_view_nereids.groovy @@ -132,8 +132,7 @@ suite("test_drop_view_nereids") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """insert into view_baseall_drop_nereids values(1,[1,2,3]);""" diff --git a/regression-test/suites/delete_p0/test_delete.groovy b/regression-test/suites/delete_p0/test_delete.groovy index 8a66e763328709..9d9df117c8745b 100644 --- a/regression-test/suites/delete_p0/test_delete.groovy +++ b/regression-test/suites/delete_p0/test_delete.groovy @@ -476,8 +476,7 @@ suite("test_delete") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy index 0800999e7d73ac..c197a138b3a4a7 100644 --- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy @@ -60,8 +60,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0,external,external "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ qt_sql """ @@ -113,8 +112,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0,external,external "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy index 27914aceb4a8d3..acb97ef955c817 100644 --- a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy +++ b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy @@ -130,8 +130,7 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent,p2"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/function_p0/test_array_map.groovy b/regression-test/suites/function_p0/test_array_map.groovy index 10274b56ed7325..95fbb8e08f216d 100644 --- a/regression-test/suites/function_p0/test_array_map.groovy +++ b/regression-test/suites/function_p0/test_array_map.groovy @@ -115,7 +115,6 @@ suite("test_array_map") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy b/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy index 07c271bec47527..a7571934f731fe 100644 --- a/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy +++ b/regression-test/suites/insert_overwrite_p0/test_iot_auto_detect_fail.groovy @@ -78,7 +78,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V1", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); @@ -146,7 +145,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V1", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_array_contains_estimate.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_array_contains_estimate.groovy index 09f2a7e891c915..c1d98f80c119a1 100644 --- a/regression-test/suites/inverted_index_p0/array_contains/test_array_contains_estimate.groovy +++ b/regression-test/suites/inverted_index_p0/array_contains/test_array_contains_estimate.groovy @@ -42,8 +42,7 @@ suite("test_array_contains_estimate", "nonConcurrent"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/inverted_index_p0/index_change/test_build_index.groovy b/regression-test/suites/inverted_index_p0/index_change/test_build_index.groovy index 66829f0337072f..0696fefe711d24 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_build_index.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_build_index.groovy @@ -140,8 +140,7 @@ suite("test_build_index", "inverted_index"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy deleted file mode 100644 index 2d267cf31b258e..00000000000000 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy +++ /dev/null @@ -1,208 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2") { - def tableName = "test_single_replica_compaction_with_format_v2" - - def backends = sql_return_maparray('show backends') - // if backens is less than 2, skip this case - if (backends.size() < 2) { - return - } - - def timeout = 60000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 - - def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" - alter_res = alter_res.toString() - if(alter_res.contains("FINISHED")) { - sleep(10000) // wait change table state to normal - logger.info(table_name + " latest alter job finished, detail: " + alter_res) - break - } - useTime = t - sleep(delta_time) - } - assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") - } - - def calc_segment_count = { tablet -> - int segment_count = 0 - String tablet_id = tablet.TabletId - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET ") - sb.append(tablet.CompactionStatus) - String command = sb.toString() - if ((context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false) { - command = command.replace("http://", "https://") + " --cert " + context.config.otherConfigs.get("trustCert") + " --cacert " + context.config.otherConfigs.get("trustCACert") + " --key " + context.config.otherConfigs.get("trustCAKey") - } - // wait for cleaning stale_rowsets - def process = command.execute() - def code = process.waitFor() - def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - def out = process.getText() - logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def tabletJson = parseJson(out.trim()) - assert tabletJson.rowsets instanceof List - for (String rowset in (List) tabletJson.rowsets) { - segment_count += Integer.parseInt(rowset.split(" ")[1]) - } - return segment_count - } - - try { - //BackendId,Cluster,IP,HeartbeatPort,BePort,HttpPort,BrpcPort,LastStartTime,LastHeartbeat,Alive,SystemDecommissioned,ClusterDecommissioned,TabletNum,DataUsedCapacity,AvailCapacity,TotalCapacity,UsedPct,MaxDiskUsedPct,Tag,ErrMsg,Version,Status - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - StringBuilder showConfigCommand = new StringBuilder(); - Boolean enableTls = (context.config.otherConfigs.get("enableTLS")?.toString()?.equalsIgnoreCase("true")) ?: false - def protocol = enableTls ? "https" : "http" - showConfigCommand.append("curl -X GET ${protocol}://") - showConfigCommand.append(backendId_to_backendIP.get(backend_id)) - showConfigCommand.append(":") - showConfigCommand.append(backendId_to_backendHttpPort.get(backend_id)) - showConfigCommand.append("/api/show_config") - if (enableTls) { - showConfigCommand.append(" --cert ${context.config.otherConfigs.get("trustCert")}") - showConfigCommand.append(" --key ${context.config.otherConfigs.get("trustCAKey")}") - showConfigCommand.append(" --cacert ${context.config.otherConfigs.get("trustCACert")}") - } - logger.info(showConfigCommand.toString()) - def process = showConfigCommand.toString().execute() - int code = process.waitFor() - String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - String out = process.getText() - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - } - } - - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `datev2` DATEV2 NOT NULL COMMENT "数据灌入日期时间", - `datetimev2_1` DATETIMEV2(3) NOT NULL COMMENT "数据灌入日期时间", - `datetimev2_2` DATETIMEV2(6) NOT NULL COMMENT "数据灌入日期时间", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", - `datetime_val1` DATETIMEV2(3) DEFAULT "1970-01-01 00:00:00.111" COMMENT "用户最后一次访问时间", - `datetime_val2` DATETIME(6) DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", - `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间", - INDEX idx_user_id (`user_id`) USING INVERTED, - INDEX idx_date (`date`) USING INVERTED, - INDEX idx_city (`city`) USING INVERTED) - DUPLICATE KEY(`user_id`, `date`, `datev2`, `datetimev2_1`, `datetimev2_2`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - PROPERTIES ( "replication_num" = "2", "disable_auto_compaction" = "true", "enable_single_replica_compaction" = "true" ); - """ - - sql """ INSERT INTO ${tableName} VALUES - (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2017-10-01 11:11:11.170000', '2017-10-01 11:11:11.110111', '2020-01-01', 1, 30, 20) - """ - - sql """ INSERT INTO ${tableName} VALUES - (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.160000', '2017-10-01 11:11:11.100111', '2020-01-02', 1, 31, 19) - """ - - sql """ INSERT INTO ${tableName} VALUES - (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Shanghai', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.150000', '2017-10-01 11:11:11.130111', '2020-01-02', 1, 31, 21) - """ - - sql """ INSERT INTO ${tableName} VALUES - (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Shanghai', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.140000', '2017-10-01 11:11:11.120111', '2020-01-03', 1, 32, 20) - """ - - sql """ INSERT INTO ${tableName} VALUES - (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Guangzhou', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.100000', '2017-10-01 11:11:11.140111', '2020-01-03', 1, 32, 22) - """ - - sql """ INSERT INTO ${tableName} VALUES - (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Guangzhou', 10, 1, '2020-01-04', '2020-01-04', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.150111', '2020-01-04', 1, 33, 21) - """ - - sql """ INSERT INTO ${tableName} VALUES - (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Shenzhen', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) - """ - - sql """ INSERT INTO ${tableName} VALUES - (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Shenzhen', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) - """ - - sql """ sync """ - sql """ set enable_common_expr_pushdown = true """ - - qt_select_default """ SELECT * FROM ${tableName} t WHERE city MATCH 'Beijing' ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ - - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus - def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - - // trigger compactions for all tablets in ${tableName} - trigger_and_wait_compaction(tableName, "full") - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - backend_id = tablet.BackendId - String ip = backendId_to_backendIP.get(backend_id) - String port = backendId_to_backendHttpPort.get(backend_id) - check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") - } - - int segmentsCount = 0 - for (def tablet in tablets) { - segmentsCount += calc_segment_count(tablet) - } - - def dedup_tablets = deduplicate_tablets(tablets) - - // In the p0 testing environment, there are no expected operations such as scaling down BE (backend) services - // if tablets or dedup_tablets is empty, exception is thrown, and case fail - int replicaNum = Math.floor(tablets.size() / dedup_tablets.size()) - if (replicaNum < 1) - { - assert(false); - } - - assert (segmentsCount <= 8*replicaNum) - qt_select_default2 """ SELECT * FROM ${tableName} t WHERE city MATCH 'Beijing' ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ - } finally { - } -} diff --git a/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy b/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy index 5a60541f4a1146..576c31cc8d5785 100644 --- a/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy +++ b/regression-test/suites/inverted_index_p0/test_arrayInvertedIdx_profile.groovy @@ -82,8 +82,7 @@ suite("test_arrayInvertedIdx_profile", "nonConcurrent"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy index 78f22b1b514e5a..d0f99d3110937e 100644 --- a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy @@ -41,8 +41,7 @@ suite("test_array_contains_with_inverted_index"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/inverted_index_p0/test_array_index.groovy b/regression-test/suites/inverted_index_p0/test_array_index.groovy index 186738ce25853f..ba6e1e1f398986 100644 --- a/regression-test/suites/inverted_index_p0/test_array_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_index.groovy @@ -56,8 +56,7 @@ suite("test_array_index1"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ set enable_common_expr_pushdown = true """ @@ -114,8 +113,7 @@ suite("test_array_index2"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy index e3255beb8a898c..d8e8575a052b30 100644 --- a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy @@ -75,8 +75,7 @@ suite("test_array_with_inverted_index_all_type"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -101,8 +100,7 @@ suite("test_array_with_inverted_index_all_type"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -145,8 +143,7 @@ suite("test_array_with_inverted_index_all_type"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -170,8 +167,7 @@ suite("test_array_with_inverted_index_all_type"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -215,7 +211,6 @@ suite("test_array_with_inverted_index_all_type"){ "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "enable_unique_key_merge_on_write" = "true" ); """ @@ -241,7 +236,6 @@ suite("test_array_with_inverted_index_all_type"){ "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "enable_unique_key_merge_on_write" = "true" ); """ diff --git a/regression-test/suites/inverted_index_p0/test_delete.groovy b/regression-test/suites/inverted_index_p0/test_delete.groovy index 06cd51cc87aa47..8cdb4cf859fc9c 100644 --- a/regression-test/suites/inverted_index_p0/test_delete.groovy +++ b/regression-test/suites/inverted_index_p0/test_delete.groovy @@ -43,8 +43,7 @@ suite("test_delete"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ set enable_common_expr_pushdown = true """ diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy index 3e54a10b326c51..d95985176f7da4 100644 --- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy @@ -132,7 +132,6 @@ suite("test_index_rqg_bug8", "test_index_rqg_bug8"){ "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy index aa7d15141b5929..cc2b871142b505 100644 --- a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy @@ -36,8 +36,7 @@ suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){ "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ set enable_profile = true;""" diff --git a/regression-test/suites/load_p0/stream_load/test_csv_split_line.groovy b/regression-test/suites/load_p0/stream_load/test_csv_split_line.groovy index 76f18317fb8ed8..ccd5c8b41f7777 100644 --- a/regression-test/suites/load_p0/stream_load/test_csv_split_line.groovy +++ b/regression-test/suites/load_p0/stream_load/test_csv_split_line.groovy @@ -142,8 +142,7 @@ suite("test_csv_split_line", "p0") { "colocate_with" = "__global__crm_user_group", "storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/load_p0/stream_load/test_csv_with_double_quotes.groovy b/regression-test/suites/load_p0/stream_load/test_csv_with_double_quotes.groovy index 03a31f7997ecad..15cb18dc3f80fd 100644 --- a/regression-test/suites/load_p0/stream_load/test_csv_with_double_quotes.groovy +++ b/regression-test/suites/load_p0/stream_load/test_csv_with_double_quotes.groovy @@ -75,8 +75,7 @@ suite("test_csv_with_double_quotes", "p0") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ } diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index 13b45bb2c65b50..79ed17b16bbcc3 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -134,8 +134,7 @@ suite("test_json_load", "p0,nonConcurrent") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ } diff --git a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy index 312905bc8687d8..e199b87cd8a509 100644 --- a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy +++ b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy @@ -32,8 +32,7 @@ suite("test_load_with_map_nested_array", "p0") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/load_p0/stream_load/test_load_with_transfer_encoding.groovy b/regression-test/suites/load_p0/stream_load/test_load_with_transfer_encoding.groovy index 2926b5e2fc019a..050f72590f9feb 100644 --- a/regression-test/suites/load_p0/stream_load/test_load_with_transfer_encoding.groovy +++ b/regression-test/suites/load_p0/stream_load/test_load_with_transfer_encoding.groovy @@ -61,7 +61,6 @@ suite("test_load_with_transfer_encoding", "p0") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "replication_num" = "1", "group_commit_data_bytes" = "134217728" diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy index c0c11aaccd4024..270b494b1f0ee6 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_error_url.groovy @@ -141,7 +141,6 @@ suite("test_stream_load_error_url", "p0") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql b/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql index c0be1514f90a62..f16742cf2ba60e 100644 --- a/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql +++ b/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql @@ -69,7 +69,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "replication_allocation" = "tag.location.default: 1" diff --git a/regression-test/suites/mv_p0/mv_with_force_drop/mv_with_force_drop.groovy b/regression-test/suites/mv_p0/mv_with_force_drop/mv_with_force_drop.groovy index b87603067e63af..d4cf179c090b53 100644 --- a/regression-test/suites/mv_p0/mv_with_force_drop/mv_with_force_drop.groovy +++ b/regression-test/suites/mv_p0/mv_with_force_drop/mv_with_force_drop.groovy @@ -38,8 +38,7 @@ suite("mv_with_force_drop") { "replication_allocation"="tag.location.default:1", "is_being_synced"="false", "storage_format"="V2", - "disable_auto_compaction"="false", - "enable_single_replica_compaction"="false" + "disable_auto_compaction"="false" ); """ diff --git a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy index df4b734216ff6d..2932f4aa7419d6 100644 --- a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy +++ b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy @@ -90,7 +90,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/mv_p0/test_tcu/test_tcu.groovy b/regression-test/suites/mv_p0/test_tcu/test_tcu.groovy index a2a6e5310a5f6a..e9d768277bb09d 100644 --- a/regression-test/suites/mv_p0/test_tcu/test_tcu.groovy +++ b/regression-test/suites/mv_p0/test_tcu/test_tcu.groovy @@ -43,7 +43,7 @@ suite ("test_tcu") { ) ENGINE = OLAP DUPLICATE KEY(`a`) COMMENT 'OLAP' PARTITION BY RANGE(`e`) ( PARTITION p20230822 VALUES - [('2023-08-22 00:00:00'), ('2023-08-23 00:00:00')), PARTITION p20230823 VALUES [('2023-08-23 00:00:00'), ('2023-08-24 00:00:00')), PARTITION p20230824 VALUES [('2023-08-24 00:00:00'), ('2023-08-25 00:00:00')), PARTITION p20230825 VALUES [('2023-08-25 00:00:00'), ('2023-08-26 00:00:00')), PARTITION p20230826 VALUES [('2023-08-26 00:00:00'), ('2023-08-27 00:00:00'))) DISTRIBUTED BY HASH(`a`) BUCKETS 20 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.time_zone" = "Etc/UTC", "dynamic_partition.start" = "-2147483648", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.replication_allocation" = "tag.location.default: 1", "dynamic_partition.buckets" = "20", "dynamic_partition.create_history_partition" = "false", "dynamic_partition.history_partition_num" = "-1", "dynamic_partition.hot_partition_num" = "0", "dynamic_partition.reserved_history_periods" = "NULL", "dynamic_partition.storage_policy" = "", "dynamic_partition.storage_medium" = "HDD", "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", "enable_single_replica_compaction" = "false" ) ; + [('2023-08-22 00:00:00'), ('2023-08-23 00:00:00')), PARTITION p20230823 VALUES [('2023-08-23 00:00:00'), ('2023-08-24 00:00:00')), PARTITION p20230824 VALUES [('2023-08-24 00:00:00'), ('2023-08-25 00:00:00')), PARTITION p20230825 VALUES [('2023-08-25 00:00:00'), ('2023-08-26 00:00:00')), PARTITION p20230826 VALUES [('2023-08-26 00:00:00'), ('2023-08-27 00:00:00'))) DISTRIBUTED BY HASH(`a`) BUCKETS 20 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.time_zone" = "Etc/UTC", "dynamic_partition.start" = "-2147483648", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.replication_allocation" = "tag.location.default: 1", "dynamic_partition.buckets" = "20", "dynamic_partition.create_history_partition" = "false", "dynamic_partition.history_partition_num" = "-1", "dynamic_partition.hot_partition_num" = "0", "dynamic_partition.reserved_history_periods" = "NULL", "dynamic_partition.storage_policy" = "", "dynamic_partition.storage_medium" = "HDD", "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false" ) ; """ sql """insert into tcu_test values('vin78215KHVB','2023-08-23 06:47:14','2023-08-23 06:47:15','2023-08-23 06:47:15','2023-08-23 00:00:00','MessageName','MessageType','referenceId','ActivityID','ProtofileVersion','{"k22":{"k221":{"k2211":2023,"k2212":8,"k2213":23,"k2214":6,"k2215":47,"k2216":14},"k222":{"k2221":84,"k2222":43,"k2223":3,"k2224":{"xxxx01_U_Actl":4.164,"xxxx02_U_Actl":4.163,"xxxx03_U_Actl":4.155,"xxxx04_U_Actl":4.164,"xxxx05_U_Actl":4.162,"xxxx06_U_Actl":4.159,"xxxx07_U_Actl":4.16,"xxxx08_U_Actl":4.162,"xxxx09_U_Actl":4.162,"xxxx10_U_Actl":4.164,"xxxx11_U_Actl":4.161,"xxxx12_U_Actl":4.162,"xxxx13_U_Actl":4.159,"xxxx14_U_Actl":4.159,"xxxx15_U_Actl":4.162,"xxxx16_U_Actl":4.163,"xxxx17_U_Actl":4.163,"xxxx18_U_Actl":4.159,"xxxx19_U_Actl":4.161,"xxxx20_U_Actl":4.159,"xxxx21_U_Actl":4.163,"xxxx22_U_Actl":4.161,"xxxx23_U_Actl":4.164,"xxxx24_U_Actl":4.161,"xxxx25_U_Actl":4.16,"xxxx26_U_Actl":4.161,"xxxx27_U_Actl":4.161,"xxxx28_U_Actl":4.164,"xxxx29_U_Actl":4.162,"xxxx30_U_Actl":4.161,"xxxx31_U_Actl":4.16,"xxxx32_U_Actl":4.162,"xxxx33_U_Actl":4.162,"xxxx34_U_Actl":4.165,"xxxx35_U_Actl":4.16,"xxxx36_U_Actl":4.163,"xxxx37_U_Actl":4.161,"xxxx38_U_Actl":4.165,"xxxx39_U_Actl":4.161,"xxxx40_U_Actl":4.162,"xxxx41_U_Actl":4.162,"xxxx42_U_Actl":4.164,"xxxx43_U_Actl":4.169,"xxxx44_U_Actl":4.167,"xxxx45_U_Actl":4.164,"xxxx46_U_Actl":4.161,"xxxx47_U_Actl":4.164,"xxxx48_U_Actl":4.161,"xxxx49_U_Actl":4.161,"xxxx50_U_Actl":4.164,"xxxx51_U_Actl":4.163,"xxxx52_U_Actl":4.164,"xxxx53_U_Actl":4.167,"xxxx54_U_Actl":4.163,"xxxx55_U_Actl":4.163,"xxxx56_U_Actl":4.165,"xxxx57_U_Actl":4.165,"xxxx58_U_Actl":4.164,"xxxx59_U_Actl":4.163,"xxxx60_U_Actl":4.163,"xxxx61_U_Actl":4.163,"xxxx62_U_Actl":4.163,"xxxx63_U_Actl":4.163,"xxxx64_U_Actl":4.164,"xxxx65_U_Actl":4.163,"xxxx66_U_Actl":4.164,"xxxx67_U_Actl":4.16,"xxxx68_U_Actl":4.163,"xxxx69_U_Actl":4.163,"xxxx70_U_Actl":4.163,"xxxx71_U_Actl":4.162,"xxxx72_U_Actl":4.162,"xxxx73_U_Actl":4.163,"xxxx74_U_Actl":4.162,"xxxx75_U_Actl":4.162,"xxxx76_U_Actl":4.163,"xxxx77_U_Actl":4.163,"xxxx78_U_Actl":4.163,"xxxx79_U_Actl":4.166,"xxxx80_U_Actl":4.164,"xxxx81_U_Actl":4.164,"xxxx82_U_Actl":4.164,"xxxx83_U_Actl":4.167,"xxxx84_U_Actl":4.169},"k2225":12,"k2226":7,"k2227":2,"k2228":{"k22281":33,"k22282":32,"k22283":32,"k22284":32,"k22285":34,"k22286":33,"k22287":35,"k22288":34,"k22289":33}},"k223":{"k2231":"Run"}}}')""" diff --git a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy index b12aef3a656f9a..a4a8d05e4a03a4 100644 --- a/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy +++ b/regression-test/suites/nereids_arith_p0/topn/accept_null.groovy @@ -38,7 +38,6 @@ suite ("accept_null") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/nereids_p0/aggregate/agg_distinct_case_when.groovy b/regression-test/suites/nereids_p0/aggregate/agg_distinct_case_when.groovy index 546586702e5f4b..e22aa3201e8a28 100644 --- a/regression-test/suites/nereids_p0/aggregate/agg_distinct_case_when.groovy +++ b/regression-test/suites/nereids_p0/aggregate/agg_distinct_case_when.groovy @@ -36,8 +36,7 @@ suite("agg_distinct_case_when") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/nereids_p0/eager_agg/load.groovy b/regression-test/suites/nereids_p0/eager_agg/load.groovy index a43d7f18d2487a..ea483c89afc18e 100644 --- a/regression-test/suites/nereids_p0/eager_agg/load.groovy +++ b/regression-test/suites/nereids_p0/eager_agg/load.groovy @@ -58,7 +58,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V3", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/nereids_p0/explain/explain_graph.groovy b/regression-test/suites/nereids_p0/explain/explain_graph.groovy index 5ffb776de99d13..410315970d4f13 100644 --- a/regression-test/suites/nereids_p0/explain/explain_graph.groovy +++ b/regression-test/suites/nereids_p0/explain/explain_graph.groovy @@ -40,7 +40,6 @@ suite("explain_graph") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ @@ -61,7 +60,6 @@ suite("explain_graph") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ // make sure "explain graph" could work with "insert select" diff --git a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy index 220ab4038c5031..109c868dc67862 100644 --- a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy +++ b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy @@ -41,8 +41,7 @@ suite("test_pushdown_explain") { "colocate_with" = "groupa1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ insert into test_lineorder values(1,2,"asd"); """ explain { @@ -90,8 +89,7 @@ suite("test_pushdown_explain") { "colocate_with" = "groupa1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ insert into test_null_columns values(1, NULL, "value1"); """ sql """ insert into test_null_columns values(2, NULL, "value2"); """ diff --git a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy index f2ff40649d24de..5453b6766ba570 100644 --- a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy +++ b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy @@ -174,7 +174,6 @@ suite("nereids_partial_update_native_insert_stmt", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" diff --git a/regression-test/suites/nereids_p0/runtime_filter/expr-target.groovy b/regression-test/suites/nereids_p0/runtime_filter/expr-target.groovy index 9480b860ac7c48..c7a1ce438d5260 100644 --- a/regression-test/suites/nereids_p0/runtime_filter/expr-target.groovy +++ b/regression-test/suites/nereids_p0/runtime_filter/expr-target.groovy @@ -35,7 +35,6 @@ suite("expr-target") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); @@ -58,7 +57,6 @@ suite("expr-target") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/nereids_p0/subquery/test_subquery.groovy b/regression-test/suites/nereids_p0/subquery/test_subquery.groovy index 82b858cf10be78..a40c6d231f8750 100644 --- a/regression-test/suites/nereids_p0/subquery/test_subquery.groovy +++ b/regression-test/suites/nereids_p0/subquery/test_subquery.groovy @@ -89,8 +89,7 @@ suite("test_subquery") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ CREATE TABLE `table_1000_undef_undef2` ( @@ -106,8 +105,7 @@ suite("test_subquery") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );""" explain { sql """ diff --git a/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy b/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy index e078081b0793a6..a574f94e36eee5 100644 --- a/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy +++ b/regression-test/suites/nereids_p0/subquery/test_subquery_in_project.groovy @@ -34,8 +34,7 @@ suite("test_subquery_in_project") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/nereids_p0/tablesample.groovy b/regression-test/suites/nereids_p0/tablesample.groovy index 4e6d17f8df395b..53274860ae8db8 100644 --- a/regression-test/suites/nereids_p0/tablesample.groovy +++ b/regression-test/suites/nereids_p0/tablesample.groovy @@ -31,8 +31,7 @@ suite("test_table_sample") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy index 9473b88fd65417..0897c3d7e51a65 100644 --- a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy +++ b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy @@ -33,8 +33,7 @@ suite("test_forbid_unknown_col_stats") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql "SET enable_nereids_planner=true" diff --git a/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy b/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy index 2e0452371316b4..3e4300af3170b8 100644 --- a/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy +++ b/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy @@ -71,7 +71,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -125,7 +124,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -166,7 +164,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -215,7 +212,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -285,7 +281,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -314,7 +309,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/nereids_p0/update/update_after_create_rollup.groovy b/regression-test/suites/nereids_p0/update/update_after_create_rollup.groovy index 7c6aca2cea5cfd..1442e235742bb8 100644 --- a/regression-test/suites/nereids_p0/update/update_after_create_rollup.groovy +++ b/regression-test/suites/nereids_p0/update/update_after_create_rollup.groovy @@ -104,7 +104,6 @@ suite('update_after_create_rollup') { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/nereids_rules_p0/cse/cse.groovy b/regression-test/suites/nereids_rules_p0/cse/cse.groovy index f059f05620caa8..5c8234fa8e0c0a 100644 --- a/regression-test/suites/nereids_rules_p0/cse/cse.groovy +++ b/regression-test/suites/nereids_rules_p0/cse/cse.groovy @@ -33,7 +33,6 @@ suite("cse") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/nereids_rules_p0/cte/test_cte_name_reuse.groovy b/regression-test/suites/nereids_rules_p0/cte/test_cte_name_reuse.groovy index 0f943db5b311d7..c146763853efd4 100644 --- a/regression-test/suites/nereids_rules_p0/cte/test_cte_name_reuse.groovy +++ b/regression-test/suites/nereids_rules_p0/cte/test_cte_name_reuse.groovy @@ -37,8 +37,7 @@ suite("test_cte_name_reuse") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy index c7b106eff2a289..23ab6f8467936e 100644 --- a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy +++ b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/one_phase.groovy @@ -32,7 +32,6 @@ suite("one_phase") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/nereids_rules_p0/grouping_sets/test_grouping_sets_combination.groovy b/regression-test/suites/nereids_rules_p0/grouping_sets/test_grouping_sets_combination.groovy index 6c6c1b7adbf272..0c996814d8f1f3 100644 --- a/regression-test/suites/nereids_rules_p0/grouping_sets/test_grouping_sets_combination.groovy +++ b/regression-test/suites/nereids_rules_p0/grouping_sets/test_grouping_sets_combination.groovy @@ -47,8 +47,7 @@ suite("test_grouping_sets_combination") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -70,8 +69,7 @@ suite("test_grouping_sets_combination") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy index 45af1df34f6d9e..8368257f7e5210 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy @@ -49,7 +49,6 @@ suite("union_all_compensate") { "storage_medium" = "hdd", "storage_format" = "V2", "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", 'replication_num' = '1' @@ -76,7 +75,6 @@ suite("union_all_compensate") { "storage_medium" = "hdd", "storage_format" = "V2", "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", 'replication_num' = '1' diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/prune_or.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/prune_or.groovy index e8c7cff145f182..a78731bea1cf68 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/prune_or.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/prune_or.groovy @@ -53,7 +53,6 @@ suite("prune_or") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy index 0614ec318cc5e6..a2c225b5c7adac 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy @@ -292,8 +292,7 @@ suite("test_multi_range_partition") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql "SET enable_nereids_planner=true" diff --git a/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy index ee638790de9a10..ca720dd1f05c10 100644 --- a/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy +++ b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy @@ -40,7 +40,6 @@ suite("rf_prune") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ @@ -60,7 +59,6 @@ suite("rf_prune") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ diff --git a/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy b/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy index 18c8553604931d..d46019d4856aa1 100644 --- a/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy +++ b/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy @@ -44,7 +44,6 @@ suite("transposeSemiJoinAgg") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ @@ -65,7 +64,6 @@ suite("transposeSemiJoinAgg") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" ); """ diff --git a/regression-test/suites/nereids_syntax_p0/join_order.groovy b/regression-test/suites/nereids_syntax_p0/join_order.groovy index 461c2bb448522a..7770951fd0350a 100644 --- a/regression-test/suites/nereids_syntax_p0/join_order.groovy +++ b/regression-test/suites/nereids_syntax_p0/join_order.groovy @@ -131,8 +131,7 @@ suite("join_order") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );""" sql """ create table test_table_t2 @@ -146,8 +145,7 @@ suite("join_order") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );""" sql """ create table test_table_t3 @@ -161,8 +159,7 @@ suite("join_order") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" );""" sql """insert into test_table_t1 values (1,null);""" diff --git a/regression-test/suites/nereids_syntax_p0/push_filter_through_ptopn.groovy b/regression-test/suites/nereids_syntax_p0/push_filter_through_ptopn.groovy index 739517182f870b..c6b1653d32ec57 100644 --- a/regression-test/suites/nereids_syntax_p0/push_filter_through_ptopn.groovy +++ b/regression-test/suites/nereids_syntax_p0/push_filter_through_ptopn.groovy @@ -35,8 +35,7 @@ suite("push_filter_through_ptopn") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy b/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy index c43f094bdb83d4..d8374311dc026c 100644 --- a/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy +++ b/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy @@ -34,8 +34,7 @@ suite("push_filter_through_window") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index ac3b255cdfd863..7cf912bf91b5fc 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -344,7 +344,6 @@ suite("test_prepared_stmt", "nonConcurrent") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/query_p0/aggregate/aggregate_not_group_null.groovy b/regression-test/suites/query_p0/aggregate/aggregate_not_group_null.groovy index 7be16a0f0a4e45..fc4d72187d9cc6 100644 --- a/regression-test/suites/query_p0/aggregate/aggregate_not_group_null.groovy +++ b/regression-test/suites/query_p0/aggregate/aggregate_not_group_null.groovy @@ -31,8 +31,7 @@ suite("aggregate_no_group_null") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """insert into t1_int values(1,1),(1,2),(1,3),(1,4),(1,null);""" diff --git a/regression-test/suites/query_p0/aggregate/array_agg.groovy b/regression-test/suites/query_p0/aggregate/array_agg.groovy index f58971a0fa95a5..c1952722048b4b 100644 --- a/regression-test/suites/query_p0/aggregate/array_agg.groovy +++ b/regression-test/suites/query_p0/aggregate/array_agg.groovy @@ -35,8 +35,7 @@ suite("array_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -54,8 +53,7 @@ suite("array_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -75,8 +73,7 @@ suite("array_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -210,8 +207,7 @@ suite("array_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/query_p0/aggregate/bitmap_agg.groovy b/regression-test/suites/query_p0/aggregate/bitmap_agg.groovy index b5a9fe87702197..49bd8b3a8e092d 100644 --- a/regression-test/suites/query_p0/aggregate/bitmap_agg.groovy +++ b/regression-test/suites/query_p0/aggregate/bitmap_agg.groovy @@ -28,8 +28,7 @@ suite("bitmap_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -56,8 +55,7 @@ suite("bitmap_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/query_p0/aggregate/map_agg.groovy b/regression-test/suites/query_p0/aggregate/map_agg.groovy index e98ce3635ee377..161d500d1d0c81 100644 --- a/regression-test/suites/query_p0/aggregate/map_agg.groovy +++ b/regression-test/suites/query_p0/aggregate/map_agg.groovy @@ -30,8 +30,7 @@ suite("map_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -68,8 +67,7 @@ suite("map_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ @@ -105,8 +103,7 @@ suite("map_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -143,8 +140,7 @@ suite("map_agg") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy index 8d4ae13b1cc3eb..1e4d3469cb238a 100644 --- a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy +++ b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy @@ -36,8 +36,7 @@ suite("map_agg_nested_insert_doris", "p0") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -53,8 +52,7 @@ suite("map_agg_nested_insert_doris", "p0") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/aggregate/map_agg_v1.groovy b/regression-test/suites/query_p0/aggregate/map_agg_v1.groovy index ddc9efa3912c0f..f8e4db519d37db 100644 --- a/regression-test/suites/query_p0/aggregate/map_agg_v1.groovy +++ b/regression-test/suites/query_p0/aggregate/map_agg_v1.groovy @@ -30,8 +30,7 @@ suite("map_agg_v1") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -68,8 +67,7 @@ suite("map_agg_v1") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ @@ -105,8 +103,7 @@ suite("map_agg_v1") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -143,8 +140,7 @@ suite("map_agg_v1") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/cache/condition_cache.groovy b/regression-test/suites/query_p0/cache/condition_cache.groovy index 0a6438b63ccc4f..e8072cf0a54da2 100755 --- a/regression-test/suites/query_p0/cache/condition_cache.groovy +++ b/regression-test/suites/query_p0/cache/condition_cache.groovy @@ -43,8 +43,7 @@ suite("condition_cache") { "storage_format" = "V2", "inverted_index_storage_format" = "V3", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ) """ @@ -77,8 +76,7 @@ suite("condition_cache") { "storage_format" = "V2", "inverted_index_storage_format" = "V3", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ) """ diff --git a/regression-test/suites/query_p0/cache/query_cache.groovy b/regression-test/suites/query_p0/cache/query_cache.groovy index fb15e34a4f5133..d79fe4af2adc96 100644 --- a/regression-test/suites/query_p0/cache/query_cache.groovy +++ b/regression-test/suites/query_p0/cache/query_cache.groovy @@ -44,7 +44,6 @@ suite("query_cache") { "inverted_index_storage_format" = "V3", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ) @@ -251,7 +250,6 @@ GROUP BY col_int_undef_signed; "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" diff --git a/regression-test/suites/query_p0/cast/test_complextype_to_json.groovy b/regression-test/suites/query_p0/cast/test_complextype_to_json.groovy index b9654311fc3b39..9a70fce330d2f8 100644 --- a/regression-test/suites/query_p0/cast/test_complextype_to_json.groovy +++ b/regression-test/suites/query_p0/cast/test_complextype_to_json.groovy @@ -51,8 +51,7 @@ suite('test_complextype_to_json', "query_p0") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/having/having.groovy b/regression-test/suites/query_p0/having/having.groovy index bbad236c973870..f023df17fecae4 100644 --- a/regression-test/suites/query_p0/having/having.groovy +++ b/regression-test/suites/query_p0/having/having.groovy @@ -38,8 +38,7 @@ suite("having", "query,p0,arrow_flight_sql") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """explain select count(*) diff --git a/regression-test/suites/query_p0/join/rqg/doris8317/doris8317.groovy b/regression-test/suites/query_p0/join/rqg/doris8317/doris8317.groovy index eb87d3803f6ccf..41af11b43a91aa 100644 --- a/regression-test/suites/query_p0/join/rqg/doris8317/doris8317.groovy +++ b/regression-test/suites/query_p0/join/rqg/doris8317/doris8317.groovy @@ -33,8 +33,7 @@ suite("doris8317") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/query_p0/join/test_collect_list_distinct.groovy b/regression-test/suites/query_p0/join/test_collect_list_distinct.groovy index 059e15180b4c74..66193f2da0d926 100644 --- a/regression-test/suites/query_p0/join/test_collect_list_distinct.groovy +++ b/regression-test/suites/query_p0/join/test_collect_list_distinct.groovy @@ -38,8 +38,7 @@ suite("test_collect_list_distinct") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ @@ -57,8 +56,7 @@ suite("test_collect_list_distinct") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ @@ -74,8 +72,7 @@ suite("test_collect_list_distinct") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/join/test_join_with_cast_array.groovy b/regression-test/suites/query_p0/join/test_join_with_cast_array.groovy index 3147d703c724be..e675a09b0463ba 100644 --- a/regression-test/suites/query_p0/join/test_join_with_cast_array.groovy +++ b/regression-test/suites/query_p0/join/test_join_with_cast_array.groovy @@ -73,8 +73,7 @@ suite("test_join_with_cast_array", "query,p0") { "binlog.enable" = "false", "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", - "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false" + "binlog.max_history_nums" = "9223372036854775807" ); """ sql "drop table if exists dwd_oa_customer" @@ -97,8 +96,7 @@ suite("test_join_with_cast_array", "query,p0") { "binlog.enable" = "false", "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", - "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false" + "binlog.max_history_nums" = "9223372036854775807" );""" sql "drop table if exists tb_media" @@ -120,8 +118,7 @@ suite("test_join_with_cast_array", "query,p0") { "binlog.enable" = "false", "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", - "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false" + "binlog.max_history_nums" = "9223372036854775807" ); """ sql "drop table if exists tb_reconciliation_detail" @@ -144,8 +141,7 @@ suite("test_join_with_cast_array", "query,p0") { "binlog.enable" = "false", "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", - "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false" + "binlog.max_history_nums" = "9223372036854775807" ); """ sql "drop table if exists tb_reconciliation_media_order" @@ -173,8 +169,7 @@ suite("test_join_with_cast_array", "query,p0") { "binlog.enable" = "false", "binlog.ttl_seconds" = "86400", "binlog.max_bytes" = "9223372036854775807", - "binlog.max_history_nums" = "9223372036854775807", - "enable_single_replica_compaction" = "false" + "binlog.max_history_nums" = "9223372036854775807" );""" def stream_load_by_name = {tableName, tblRows -> diff --git a/regression-test/suites/query_p0/operator/test_sort_operator.groovy b/regression-test/suites/query_p0/operator/test_sort_operator.groovy index 60aa75b16edbd8..2d6ea5af4960be 100644 --- a/regression-test/suites/query_p0/operator/test_sort_operator.groovy +++ b/regression-test/suites/query_p0/operator/test_sort_operator.groovy @@ -35,8 +35,7 @@ suite("test_sort_operator", "query,p0,arrow_flight_sql") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/set/rqg_prod_20240226/rqg_prod_20240226.groovy b/regression-test/suites/query_p0/set/rqg_prod_20240226/rqg_prod_20240226.groovy index 117b3dfcb3e912..580e9381771802 100644 --- a/regression-test/suites/query_p0/set/rqg_prod_20240226/rqg_prod_20240226.groovy +++ b/regression-test/suites/query_p0/set/rqg_prod_20240226/rqg_prod_20240226.groovy @@ -69,7 +69,6 @@ PROPERTIES ( "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" @@ -131,7 +130,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions2.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions2.groovy index 3d9258f656801f..d150b976d03a7b 100644 --- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions2.groovy +++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions2.groovy @@ -161,8 +161,7 @@ suite("test_aggregate_all_functions2") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy index 051c65c3730768..8c68c6e46a1cd3 100644 --- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy +++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy @@ -33,8 +33,7 @@ suite("test_count_distinct_with_collection_type", "p0") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ test { diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/sql/q04.sql b/regression-test/suites/query_p0/sql_functions/array_functions/sql/q04.sql index 0e26d95d0606a6..c2629592f475a5 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/sql/q04.sql +++ b/regression-test/suites/query_p0/sql_functions/array_functions/sql/q04.sql @@ -49,8 +49,7 @@ PROPERTIES ( "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", -"disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false" +"disable_auto_compaction" = "false" ); INSERT INTO ads_pi_cuser_all_info diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy index a7737f68a31b7d..2c1c86e9475917 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy @@ -137,8 +137,7 @@ suite("test_array_map_function") { "storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy index 51b8dcf4ec6675..515c0e286a9c29 100644 --- a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy +++ b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy @@ -63,8 +63,7 @@ suite("explode") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql "insert into baseall_explode_numbers values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15);" @@ -89,8 +88,7 @@ suite("explode") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ @@ -158,7 +156,6 @@ suite("explode") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy index 5f596bdd40f5b7..e4124faa2ba502 100644 --- a/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy +++ b/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy @@ -33,7 +33,6 @@ suite("explode_array_decimal") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000" );""" diff --git a/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy b/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy index 540a6a0894171d..e37356f98e0a46 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy @@ -36,8 +36,7 @@ suite("test_query_sys_rowsets", "query,p0") { DISTRIBUTED BY HASH(`a`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", - "disable_auto_compaction" = "true", - "enable_single_replica_compaction"="true" + "disable_auto_compaction" = "true" ); """ diff --git a/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy index 48281198cfd4f7..181ccd894521ce 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_scan_rowsets.groovy @@ -42,8 +42,7 @@ suite("test_query_sys_scan_rowsets", "query,p0") { DISTRIBUTED BY HASH(`a`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", - "disable_auto_compaction" = "true", - "enable_single_replica_compaction"="true" + "disable_auto_compaction" = "true" ); """ diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 72d3a8391c0d18..8ad5e85f9ef3ef 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -124,8 +124,7 @@ suite("test_query_sys_tables", "query,p0") { DISTRIBUTED BY HASH(`a`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", - "disable_auto_compaction" = "true", - "enable_single_replica_compaction"="true" + "disable_auto_compaction" = "true" );""" sql """ GRANT SELECT_PRIV,ALTER_PRIV,LOAD_PRIV ON table_privileges_demo.test_table_privileges TO 'cywtable'@'%'; """ diff --git a/regression-test/suites/query_p0/view/test_view2.groovy b/regression-test/suites/query_p0/view/test_view2.groovy index d28ad7f0ca864c..3c592e86dfdbe1 100644 --- a/regression-test/suites/query_p0/view/test_view2.groovy +++ b/regression-test/suites/query_p0/view/test_view2.groovy @@ -37,8 +37,7 @@ suite("test_view2") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql """ @@ -61,8 +60,7 @@ suite("test_view2") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/query_p0/virtual_slot_ref/char_type_shrink_before_project.groovy b/regression-test/suites/query_p0/virtual_slot_ref/char_type_shrink_before_project.groovy index 67c9c34372afff..63fe450b92c270 100644 --- a/regression-test/suites/query_p0/virtual_slot_ref/char_type_shrink_before_project.groovy +++ b/regression-test/suites/query_p0/virtual_slot_ref/char_type_shrink_before_project.groovy @@ -34,7 +34,6 @@ suite("char_type_shrink_before_projec") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/row_store/load.groovy b/regression-test/suites/row_store/load.groovy index e47b38bc421403..68b4428a2d6904 100644 --- a/regression-test/suites/row_store/load.groovy +++ b/regression-test/suites/row_store/load.groovy @@ -35,8 +35,7 @@ suite("test_row_store", "p0") { "storage_format" = "V2", "light_schema_change" = "true", "store_row_column" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql "insert into ${testTable} (tag,tag_value,user_ids) values ('10001','23',34.234),('10001','23',34.234);" @@ -60,8 +59,7 @@ suite("test_row_store", "p0") { "storage_format" = "V2", "light_schema_change" = "true", "store_row_column" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ sql "insert into ${testTable} (tag,tag_value,user_ids) values ('10001','23',34.234),('10001','23',34.234);" diff --git a/regression-test/suites/schema_change/test_alter_table_property.groovy b/regression-test/suites/schema_change/test_alter_table_property.groovy index f2383b800c6c5c..2cab3584777ee8 100644 --- a/regression-test/suites/schema_change/test_alter_table_property.groovy +++ b/regression-test/suites/schema_change/test_alter_table_property.groovy @@ -35,18 +35,8 @@ suite("test_alter_table_property") { def showResult1 = sql """show create table ${tableName}""" logger.info("${showResult1}") - assertTrue(showResult1.toString().containsIgnoreCase('"enable_single_replica_compaction" = "false"')) if (!isCloudMode()) { - sql """ - alter table ${tableName} set ("enable_single_replica_compaction" = "true") - """ - sql """sync""" - - def showResult2 = sql """show create table ${tableName}""" - logger.info("${showResult2}") - assertTrue(showResult2.toString().containsIgnoreCase('"enable_single_replica_compaction" = "true"')) - test { sql """ alter table ${tableName} set ("file_cache_ttl_seconds" = "86400") @@ -54,13 +44,6 @@ suite("test_alter_table_property") { // exception "modifying property [file_cache_ttl_seconds] is forbidden" exception "Cann't modify property 'file_cache_ttl_seconds'" } - } else { - test { - sql """ - alter table ${tableName} set ("enable_single_replica_compaction" = "true") - """ - exception "modifying property [enable_single_replica_compaction] is forbidden" - } } assertTrue(showResult1.toString().containsIgnoreCase('"disable_auto_compaction" = "false"')) diff --git a/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy b/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy index 6beb145a10e269..8345861b50095f 100644 --- a/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy +++ b/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy @@ -73,7 +73,6 @@ suite("test_alter_rollup_table") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy b/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy index 873cf2d4c0a9cb..65122f234ca731 100644 --- a/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy +++ b/regression-test/suites/schema_change_p0/test_schema_change_delete.groovy @@ -44,7 +44,6 @@ suite("test_schema_change_delete") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "true", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ) diff --git a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy index 9a568abc896484..9194984c8543bc 100644 --- a/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_with_storage_policy.groovy @@ -81,8 +81,7 @@ suite("test_show_create_table_with_storage_policy") { "dynamic_partition.storage_policy" = "${storagePolicyName}", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 46aa9adb529296..2f715c92f557ef 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -1270,8 +1270,7 @@ PARTITION `p599` VALUES IN (599) "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ diff --git a/regression-test/suites/statistics/test_scale_num_nulls.groovy b/regression-test/suites/statistics/test_scale_num_nulls.groovy index d1770d0acd5925..043b072ca62acc 100644 --- a/regression-test/suites/statistics/test_scale_num_nulls.groovy +++ b/regression-test/suites/statistics/test_scale_num_nulls.groovy @@ -40,7 +40,6 @@ suite("test_scale_num_nulls") { "inverted_index_storage_format" = "V3", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy b/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy index c77e74e0b6c3af..ee28436e4077b0 100644 --- a/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy +++ b/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy @@ -42,8 +42,7 @@ suite("map_uniq_with_local_tvf", "p0") { "is_being_synced" = "false", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "disable_auto_compaction" = "false" ); """ qt_sql """ diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy index 65340ee066e2c3..8cf232ad5b935b 100644 --- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy +++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_native_insert_stmt.groovy @@ -183,7 +183,6 @@ suite("test_partial_update_native_insert_stmt", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy index cf86a0a36fd1e5..0fa587fc32f147 100644 --- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy +++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_strict_mode.groovy @@ -51,7 +51,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName} order by id;""" @@ -100,7 +99,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName2} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName2} order by id;""" @@ -148,7 +146,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName3} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName3} order by id;""" @@ -197,7 +194,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName4} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" sql """insert into ${tableName4} values(3,"steve",23,"beijing",500,"2023-07-03 12:00:02");""" @@ -244,7 +240,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" sql """insert into ${tableName5} values(3,"steve",23,"beijing",500,"2023-07-03 12:00:02");""" diff --git a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy index 747f09bf2a6161..be6f66ebdd031b 100644 --- a/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy +++ b/regression-test/suites/unique_with_mow_c_p0/partial_update/test_partial_update_upsert.groovy @@ -48,7 +48,6 @@ suite("test_partial_update_upsert", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName} order by id;""" @@ -87,7 +86,6 @@ suite("test_partial_update_upsert", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName2} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName2} order by id;""" diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy index 87eb39a3b5b567..b7fdaaee8869f1 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy @@ -170,7 +170,6 @@ suite("test_partial_update_native_insert_stmt", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy index dc591d7b02a6c4..b634e6cbaaac2e 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy @@ -48,7 +48,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName} order by id;""" @@ -97,7 +96,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName2} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName2} order by id;""" @@ -145,7 +143,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName3} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName3} order by id;""" @@ -194,7 +191,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName4} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" sql """insert into ${tableName4} values(3,"steve",23,"beijing",500,"2023-07-03 12:00:02");""" @@ -241,7 +237,6 @@ suite("test_partial_update_strict_mode", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" sql """insert into ${tableName5} values(3,"steve",23,"beijing",500,"2023-07-03 12:00:02");""" diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy index ba12be2b99ae2d..bc80d3940f5bfd 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_upsert.groovy @@ -45,7 +45,6 @@ suite("test_partial_update_upsert", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName} order by id;""" @@ -84,7 +83,6 @@ suite("test_partial_update_upsert", "p0") { "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName2} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");""" qt_sql """select * from ${tableName2} order by id;""" diff --git a/regression-test/suites/unique_with_mow_p0/test_create_table.groovy b/regression-test/suites/unique_with_mow_p0/test_create_table.groovy index 0336beb7c039f6..5326672db2cdd8 100644 --- a/regression-test/suites/unique_with_mow_p0/test_create_table.groovy +++ b/regression-test/suites/unique_with_mow_p0/test_create_table.groovy @@ -94,24 +94,4 @@ suite("test_create_table") { """ exception "enable_unique_key_merge_on_write property only support unique key table" } - - // unique table with enable_unique_key_merge_on_write and enable_single_replica_compaction property - test { - sql """ - CREATE TABLE `$tableName` ( - `c_custkey` int(11) NOT NULL COMMENT "", - `c_name` varchar(26) NOT NULL COMMENT "", - `c_address` varchar(41) NOT NULL COMMENT "", - `c_city` varchar(11) NOT NULL COMMENT "" - ) - UNIQUE KEY (`c_custkey`) - DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true", - "enable_single_replica_compaction" = "true" - ); - """ - exception "enable_single_replica_compaction property is not supported for merge-on-write table" - } } diff --git a/regression-test/suites/variant_p0/doc_mode/load.groovy b/regression-test/suites/variant_p0/doc_mode/load.groovy index 11ab4b8108ffd6..896ee0d8efeac3 100644 --- a/regression-test/suites/variant_p0/doc_mode/load.groovy +++ b/regression-test/suites/variant_p0/doc_mode/load.groovy @@ -381,7 +381,6 @@ suite("regression_test_variant_doc_value", "p0"){ "light_schema_change" = "true", "store_row_column" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy b/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy index f4bde164b4367a..19dd98f159f14b 100644 --- a/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy +++ b/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy @@ -268,7 +268,6 @@ suite("test_variant_predefine_doc_value", "nonConcurrent"){ "store_row_column" = "true", "row_store_page_size" = "16384", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/variant_p0/doc_mode/predefine/test_predefine_type_index.groovy b/regression-test/suites/variant_p0/doc_mode/predefine/test_predefine_type_index.groovy index 057f4ed0f7ad04..3d13529ea27b2f 100644 --- a/regression-test/suites/variant_p0/doc_mode/predefine/test_predefine_type_index.groovy +++ b/regression-test/suites/variant_p0/doc_mode/predefine/test_predefine_type_index.groovy @@ -89,7 +89,6 @@ suite("variant_predefine_index_doc_value", "p0"){ "storage_format" = "V2", "inverted_index_storage_format" = "V2", "light_schema_change" = "true", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "disable_auto_compaction" = "true" diff --git a/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy b/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy index 3198d02045bf7b..db0f4ea9661218 100644 --- a/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy +++ b/regression-test/suites/variant_p0/doc_mode/with_index/test_array_contains_with_inverted_index.groovy @@ -50,7 +50,6 @@ suite("array_contains_with_index_doc_value", "p0") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "inverted_index_storage_format" = "$storageFormat" ); """ diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index f7b7370a52b720..0f88a264dc693f 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -379,7 +379,6 @@ suite("regression_test_variant", "p0"){ "light_schema_change" = "true", "store_row_column" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy b/regression-test/suites/variant_p0/mv/multi_slot.groovy index d4e46fe0ae94ff..11253d7f29aa87 100644 --- a/regression-test/suites/variant_p0/mv/multi_slot.groovy +++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy @@ -67,7 +67,6 @@ suite ("multi_slot") { "inverted_index_storage_format" = "V1", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "replication_num" = "1" ); """ diff --git a/regression-test/suites/variant_p0/predefine/load.groovy b/regression-test/suites/variant_p0/predefine/load.groovy index a2fb8b9ea4b7fc..7ab926a7ab316f 100644 --- a/regression-test/suites/variant_p0/predefine/load.groovy +++ b/regression-test/suites/variant_p0/predefine/load.groovy @@ -272,7 +272,6 @@ suite("regression_test_variant_predefine_schema", "p0"){ "store_row_column" = "true", "row_store_page_size" = "16384", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_type_index.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_type_index.groovy index 6fb0872b4976d8..19071d0b42a14d 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_type_index.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_type_index.groovy @@ -88,7 +88,6 @@ suite("test_variant_predefine_index_type", "p0"){ "storage_format" = "V2", "inverted_index_storage_format" = "V2", "light_schema_change" = "true", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "disable_auto_compaction" = "true" diff --git a/regression-test/suites/variant_p0/rqg/fix_rqg1.groovy b/regression-test/suites/variant_p0/rqg/fix_rqg1.groovy index e66b2795afc9ed..98bd07d8fa97ca 100644 --- a/regression-test/suites/variant_p0/rqg/fix_rqg1.groovy +++ b/regression-test/suites/variant_p0/rqg/fix_rqg1.groovy @@ -35,7 +35,6 @@ suite("variant_rqg_fix1", "p0") { "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); diff --git a/regression-test/suites/variant_p0/rqg/rqg6.sql b/regression-test/suites/variant_p0/rqg/rqg6.sql index dd9adcb148a898..0caf7b5f64e14c 100644 --- a/regression-test/suites/variant_p0/rqg/rqg6.sql +++ b/regression-test/suites/variant_p0/rqg/rqg6.sql @@ -37,7 +37,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );; diff --git a/regression-test/suites/variant_p0/rqg/rqg7.sql b/regression-test/suites/variant_p0/rqg/rqg7.sql index 5871881de82f78..05099b9329ce97 100644 --- a/regression-test/suites/variant_p0/rqg/rqg7.sql +++ b/regression-test/suites/variant_p0/rqg/rqg7.sql @@ -57,7 +57,6 @@ PROPERTIES ( "light_schema_change" = "true", "replication_num" = "1", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );; @@ -125,7 +124,6 @@ PROPERTIES ( "light_schema_change" = "true", "replication_num" = "1", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );; @@ -193,7 +191,6 @@ PROPERTIES ( "light_schema_change" = "true", "replication_num" = "1", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );; @@ -261,7 +258,6 @@ PROPERTIES ( "light_schema_change" = "true", "replication_num" = "1", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );; diff --git a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy index 32036d895386d1..fcbdadf9064785 100644 --- a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy +++ b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy @@ -50,7 +50,6 @@ suite("test_array_contains_with_inverted_index") { "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "inverted_index_storage_format" = "$storageFormat" ); """ diff --git a/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy b/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy index 867810003d680c..6494fa3bc78858 100644 --- a/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy +++ b/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy @@ -104,7 +104,6 @@ suite("test_variant_arrayInvertedIdx_profile", "p0,nonConcurrent"){ "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", "inverted_index_storage_format" = "$storageFormat" ); """ diff --git a/tools/coffeebench-tools/ddl/create-tables.sql b/tools/coffeebench-tools/ddl/create-tables.sql index e2c1c11deb94e2..ccc83d82f1e7d9 100644 --- a/tools/coffeebench-tools/ddl/create-tables.sql +++ b/tools/coffeebench-tools/ddl/create-tables.sql @@ -92,7 +92,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); @@ -114,7 +113,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" ); @@ -138,7 +136,6 @@ PROPERTIES ( "inverted_index_storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false", -"enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728" );