diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 65cc587ac1899d..4723e65ac932d0 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -72,6 +72,8 @@ #include "runtime/snapshot_loader.h" #include "runtime/user_function_cache.h" #include "service/backend_options.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" +#include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/data_dir.h" #include "storage/olap_common.h" @@ -911,10 +913,11 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest& } if (tablet_meta_info.__isset.compaction_policy) { if (tablet_meta_info.compaction_policy != CUMULATIVE_SIZE_BASED_POLICY && - tablet_meta_info.compaction_policy != CUMULATIVE_TIME_SERIES_POLICY) { + tablet_meta_info.compaction_policy != CUMULATIVE_TIME_SERIES_POLICY && + tablet_meta_info.compaction_policy != CUMULATIVE_BINLOG_POLICY) { status = Status::InvalidArgument( "invalid compaction policy, only support for size_based or " - "time_series"); + "time_series or binlog"); continue; } tablet->tablet_meta()->set_compaction_policy(tablet_meta_info.compaction_policy); @@ -1604,8 +1607,6 @@ void submit_table_compaction_callback(StorageEngine& engine, const TAgentTaskReq compaction_type = CompactionType::CUMULATIVE_COMPACTION; } else if (compaction_req.type == "full") { compaction_type = CompactionType::FULL_COMPACTION; - } else if (compaction_req.type == "binlog") { - compaction_type = CompactionType::BINLOG_COMPACTION; } else { LOG(WARNING) << "unknown compaction type: " << compaction_req.type << ", tablet_id=" << compaction_req.tablet_id; @@ -1633,17 +1634,6 @@ void submit_table_compaction_callback(StorageEngine& engine, const TAgentTaskReq return; } - if (compaction_type == CompactionType::BINLOG_COMPACTION) { - Status status = engine.submit_compaction_task(tablet_ptr, CompactionType::BINLOG_COMPACTION, - /*force=*/false, /*eager=*/true, - /*trigger_method=*/1); - if (!status.ok()) { - LOG(WARNING) << "failed to submit binlog compaction task. tablet_id=" - << tablet_ptr->tablet_id() << ", error=" << status; - } - return; - } - // base / cumulative auto* data_dir = tablet_ptr->data_dir(); if (!tablet_ptr->can_do_compaction(data_dir->path_hash(), compaction_type)) { diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index d136944ae4b4ea..88da8974816782 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -537,7 +537,8 @@ Status CloudCumulativeCompaction::pick_rowsets_to_compact() { _engine.cumu_compaction_policy(compaction_policy) ->pick_input_rowsets(cloud_tablet(), candidate_rowsets, max_score, config::cumulative_compaction_min_deltas, &_input_rowsets, - &_last_delete_version, &compaction_score); + &_last_delete_version, &compaction_score, + _allow_delete_in_cumu_compaction); if (_input_rowsets.empty()) { return Status::Error( diff --git a/be/src/cloud/cloud_cumulative_compaction.h b/be/src/cloud/cloud_cumulative_compaction.h index 6c0570faa4f5b7..2a97ccf98eaf3b 100644 --- a/be/src/cloud/cloud_cumulative_compaction.h +++ b/be/src/cloud/cloud_cumulative_compaction.h @@ -52,14 +52,14 @@ class CloudCumulativeCompaction : public CloudCompactionMixin { std::string_view compaction_name() const override { return "CloudCumulativeCompaction"; } + ReaderType compaction_type() const override { return ReaderType::READER_CUMULATIVE_COMPACTION; } + Status modify_rowsets() override; Status garbage_collection() override; void update_cumulative_point(); - ReaderType compaction_type() const override { return ReaderType::READER_CUMULATIVE_COMPACTION; } - int64_t _input_segments = 0; int64_t _max_conflict_version = 0; // Snapshot values when pick input rowsets diff --git a/be/src/cloud/cloud_cumulative_compaction_binlog_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_binlog_policy.cpp new file mode 100644 index 00000000000000..5a385fea647a13 --- /dev/null +++ b/be/src/cloud/cloud_cumulative_compaction_binlog_policy.cpp @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cloud/cloud_cumulative_compaction_binlog_policy.h" + +#include "cloud/cloud_tablet.h" +#include "common/config.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" +#include "storage/tablet/tablet.h" +#include "util/time.h" + +namespace doris { + +bool CloudBinlogCumulativeCompactionPolicy::is_compaction_enough( + const RowsetMetaSharedPtr& rowset_meta) const { + if (rowset_meta->compaction_level() != + BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1) { + return false; + } + if (rowset_meta->start_version() == 0) { + return true; + } + return rowset_meta->data_disk_size() >= + config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || + rowset_meta->get_compaction_score() >= config::binlog_compaction_file_count_threshold; +} + +int64_t CloudBinlogCumulativeCompactionPolicy::new_cumulative_point( + CloudTablet* tablet, const RowsetSharedPtr& output_rowset, Version& last_delete_version, + int64_t last_cumulative_point) { + if (output_rowset->num_segments() == 0 || !is_compaction_enough(output_rowset->rowset_meta())) { + return last_cumulative_point; + } + return output_rowset->end_version() + 1; +} + +int64_t CloudBinlogCumulativeCompactionPolicy::get_compaction_level( + CloudTablet* tablet, const std::vector& input_rowsets, + RowsetSharedPtr output_rowset) { + DCHECK(!input_rowsets.empty()) << "tablet=" << tablet->tablet_id(); + int64_t first_level = input_rowsets.front()->rowset_meta()->compaction_level(); + for (size_t i = 1; i < input_rowsets.size(); ++i) { + DCHECK_EQ(first_level, input_rowsets[i]->rowset_meta()->compaction_level()) + << "tablet=" << tablet->tablet_id(); + DCHECK_EQ(input_rowsets[i]->start_version(), input_rowsets[i - 1]->end_version() + 1) + << "tablet=" << tablet->tablet_id(); + } + + if (first_level == BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1) { + return first_level; + } + return first_level + 1; +} + +uint32_t CloudBinlogCumulativeCompactionPolicy::calc_binlog_compaction_level_score( + CloudTablet* tablet, const std::vector& candidate_rowsets, + int8_t level) const { + uint32_t score = 0; + const int64_t point = tablet->cumulative_layer_point(); + for (const auto& rs : candidate_rowsets) { + auto rs_meta = rs->rowset_meta(); + if (rs_meta->compaction_level() != level) { + continue; + } + if (level == BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1 && + point != Tablet::K_INVALID_CUMULATIVE_POINT && rs_meta->end_version() < point) { + score += 1; + continue; + } + score += rs_meta->get_compaction_score(); + } + return score; +} + +uint32_t CloudBinlogCumulativeCompactionPolicy::calc_binlog_compaction_score( + CloudTablet* tablet, const std::vector& candidate_rowsets, + int8_t* compaction_level) const { + uint32_t max_score = 0; + int8_t max_level = -1; + for (int8_t level = 0; level < BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel; + ++level) { + uint32_t score = calc_binlog_compaction_level_score(tablet, candidate_rowsets, level); + if (score > max_score) { + max_score = score; + max_level = level; + } + } + if (compaction_level != nullptr) { + *compaction_level = max_level; + } + return max_score; +} + +void CloudBinlogCumulativeCompactionPolicy::filter_new_visible_rowsets( + const std::vector& candidate_rowsets, + std::vector* output_rowsets) const { + output_rowsets->clear(); + int64_t now = UnixSeconds(); + int64_t max_processable_old_version = 0; + bool filter_new_rowset = + candidate_rowsets.size() <= config::binlog_compaction_file_count_threshold; + for (const auto& rs : candidate_rowsets) { + if (filter_new_rowset && rs->rowset_meta()->is_singleton_delta() && + rs->rowset_meta()->newest_write_timestamp() + + config::binlog_compaction_wait_timesec_after_visible > + now) { + continue; + } + max_processable_old_version = std::max(max_processable_old_version, rs->end_version()); + } + + output_rowsets->reserve(candidate_rowsets.size()); + for (const auto& rs : candidate_rowsets) { + if (rs->start_version() <= max_processable_old_version) { + output_rowsets->push_back(rs); + } + } +} + +int64_t CloudBinlogCumulativeCompactionPolicy::pick_input_rowsets( + CloudTablet* tablet, const std::vector& candidate_rowsets, + const int64_t max_compaction_score, const int64_t min_compaction_score, + std::vector* input_rowsets, Version* last_delete_version, + size_t* compaction_score, bool allow_delete) { + std::vector filtered_rowsets; + filter_new_visible_rowsets(candidate_rowsets, &filtered_rowsets); + + int8_t compaction_level = -1; + calc_binlog_compaction_score(tablet, filtered_rowsets, &compaction_level); + if (compaction_level < 0) { + *compaction_score = 0; + return 0; + } + + std::vector level_rowsets; + level_rowsets.reserve(filtered_rowsets.size()); + for (const auto& rs : filtered_rowsets) { + if (rs->rowset_meta()->compaction_level() != compaction_level) { + continue; + } + if (!level_rowsets.empty() && + rs->start_version() != level_rowsets.back()->end_version() + 1) { + LOG(WARNING) << "rowset is non-continuous in the same compaction_level of binlog " + "compaction. tablet=" + << tablet->tablet_id() + << ", compaction_level=" << static_cast(compaction_level) + << ", prev_version=" << level_rowsets.back()->version() + << ", next_version=" << rs->version(); + *compaction_score = 0; + return 0; + } + level_rowsets.push_back(rs); + } + + std::vector remaining_rowsets; + remaining_rowsets.reserve(level_rowsets.size()); + const int64_t point = tablet->cumulative_layer_point(); + for (const auto& rs : level_rowsets) { + if (compaction_level == BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1 && + point != Tablet::K_INVALID_CUMULATIVE_POINT && rs->end_version() < point) { + continue; + } + remaining_rowsets.push_back(rs); + } + + std::vector picked_rowsets; + picked_rowsets.reserve(remaining_rowsets.size()); + int transient_size = 0; + int64_t total_size = 0; + int64_t picked_score = 0; + for (const auto& rs : remaining_rowsets) { + if (transient_size >= max_compaction_score) { + break; + } + picked_rowsets.push_back(rs); + ++transient_size; + total_size += rs->data_disk_size(); + picked_score += rs->rowset_meta()->get_compaction_score(); + } + + bool can_do_binlog_compaction = + total_size >= config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || + picked_score >= config::binlog_compaction_file_count_threshold || + (UnixMillis() - tablet->last_cumu_compaction_success_time()) / 1000 >= + config::binlog_compaction_time_threshold_seconds; + if (transient_size < 2) { + can_do_binlog_compaction = false; + } + if (can_do_binlog_compaction) { + input_rowsets->swap(picked_rowsets); + *compaction_score = picked_score; + return transient_size; + } + + input_rowsets->clear(); + *compaction_score = 0; + return 0; +} + +} // namespace doris diff --git a/be/src/cloud/cloud_cumulative_compaction_binlog_policy.h b/be/src/cloud/cloud_cumulative_compaction_binlog_policy.h new file mode 100644 index 00000000000000..9d170c3d2e4b76 --- /dev/null +++ b/be/src/cloud/cloud_cumulative_compaction_binlog_policy.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "cloud/cloud_cumulative_compaction_policy.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" + +namespace doris { + +class CloudBinlogCumulativeCompactionPolicy final : public CloudCumulativeCompactionPolicy { +public: + CloudBinlogCumulativeCompactionPolicy() = default; + ~CloudBinlogCumulativeCompactionPolicy() override = default; + + int64_t new_cumulative_point(CloudTablet* tablet, const RowsetSharedPtr& output_rowset, + Version& last_delete_version, + int64_t last_cumulative_point) override; + + int64_t get_compaction_level(CloudTablet* tablet, + const std::vector& input_rowsets, + RowsetSharedPtr output_rowset) override; + + int64_t pick_input_rowsets(CloudTablet* tablet, + const std::vector& candidate_rowsets, + const int64_t max_compaction_score, + const int64_t min_compaction_score, + std::vector* input_rowsets, + Version* last_delete_version, size_t* compaction_score, + bool allow_delete = false) override; + + std::string name() override { return std::string(CUMULATIVE_BINLOG_POLICY); } + +private: + bool is_compaction_enough(const RowsetMetaSharedPtr& rowset_meta) const; + void filter_new_visible_rowsets(const std::vector& candidate_rowsets, + std::vector* output_rowsets) const; + uint32_t calc_binlog_compaction_level_score( + CloudTablet* tablet, const std::vector& candidate_rowsets, + int8_t level) const; + uint32_t calc_binlog_compaction_score(CloudTablet* tablet, + const std::vector& candidate_rowsets, + int8_t* compaction_level) const; +}; + +} // namespace doris diff --git a/be/src/cloud/cloud_delete_task.cpp b/be/src/cloud/cloud_delete_task.cpp index 6ad6fd9d65ee7f..e6f07bcd1c6e1a 100644 --- a/be/src/cloud/cloud_delete_task.cpp +++ b/be/src/cloud/cloud_delete_task.cpp @@ -122,7 +122,7 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine, const TPushReq& requ request.transaction_id, tablet->tablet_id(), delete_bitmap, rowset_ids, rowset, request.timeout, nullptr); } else { - if (config::enable_cloud_make_rs_visible_on_be) { + if (config::enable_cloud_make_rs_visible_on_be && !tablet->tablet_schema()->enable_tso()) { engine.meta_mgr().cache_committed_rowset(rowset->rowset_meta(), context.txn_expiration); } } diff --git a/be/src/cloud/cloud_delta_writer.cpp b/be/src/cloud/cloud_delta_writer.cpp index b936af0db2e6a0..9e7641aca1fb4f 100644 --- a/be/src/cloud/cloud_delta_writer.cpp +++ b/be/src/cloud/cloud_delta_writer.cpp @@ -33,11 +33,24 @@ bvar::Adder g_cloud_commit_empty_rowset_count("cloud_commit_empty_rowse CloudDeltaWriter::CloudDeltaWriter(CloudStorageEngine& engine, const WriteRequest& req, RuntimeProfile* profile, const UniqueId& load_id) - : BaseDeltaWriter(req, profile, load_id), _engine(engine) { + : BaseDeltaWriter(req, profile, load_id) { _rowset_builder = std::make_unique(engine, req, profile); _resource_ctx = thread_context()->resource_ctx(); } +CloudDeltaWriter::CloudDeltaWriter(CloudStorageEngine& engine, const WriteRequest& group_build_req, + const WriteRequest& sub_data_req, + const WriteRequest& sub_row_binlog_req, RuntimeProfile* profile, + const UniqueId& load_id) + : BaseDeltaWriter(group_build_req, profile, load_id) { + DCHECK(group_build_req.write_req_type == WriteRequestType::GROUP && + sub_data_req.write_req_type == WriteRequestType::DATA && + sub_row_binlog_req.write_req_type == WriteRequestType::ROW_BINLOG); + _rowset_builder = std::make_unique( + engine, group_build_req, sub_data_req, sub_row_binlog_req, profile); + _resource_ctx = thread_context()->resource_ctx(); +} + CloudDeltaWriter::~CloudDeltaWriter() = default; Status CloudDeltaWriter::batch_init(std::vector writers) { @@ -124,10 +137,6 @@ void CloudDeltaWriter::update_tablet_stats() { rowset_builder()->update_tablet_stats(); } -const RowsetMetaSharedPtr& CloudDeltaWriter::rowset_meta() { - return rowset_builder()->rowset_meta(); -} - Status CloudDeltaWriter::commit_rowset() { g_cloud_commit_rowset_count << 1; std::lock_guard lock(_mtx); @@ -138,9 +147,7 @@ Status CloudDeltaWriter::commit_rowset() { return _commit_empty_rowset(); } - // Handle normal rowset with data - return _engine.meta_mgr().commit_rowset(*rowset_meta(), "", - _rowset_builder->tablet()->table_id()); + return rowset_builder()->commit_rowset("", _rowset_builder->tablet()->table_id()); } Status CloudDeltaWriter::_commit_empty_rowset() { @@ -158,8 +165,7 @@ Status CloudDeltaWriter::_commit_empty_rowset() { return Status::OK(); } // write a empty rowset kv to keep version continuous - return _engine.meta_mgr().commit_rowset(*rowset_meta(), "", - _rowset_builder->tablet()->table_id()); + return rowset_builder()->commit_rowset("", _rowset_builder->tablet()->table_id()); } Status CloudDeltaWriter::set_txn_related_info() { diff --git a/be/src/cloud/cloud_delta_writer.h b/be/src/cloud/cloud_delta_writer.h index 3631104c23e7ae..00f6225a52b571 100644 --- a/be/src/cloud/cloud_delta_writer.h +++ b/be/src/cloud/cloud_delta_writer.h @@ -31,6 +31,9 @@ class CloudDeltaWriter final : public BaseDeltaWriter { public: CloudDeltaWriter(CloudStorageEngine& engine, const WriteRequest& req, RuntimeProfile* profile, const UniqueId& load_id); + CloudDeltaWriter(CloudStorageEngine& engine, const WriteRequest& group_build_req, + const WriteRequest& sub_data_req, const WriteRequest& sub_row_binlog_req, + RuntimeProfile* profile, const UniqueId& load_id); ~CloudDeltaWriter() override; Status write(const Block* block, const TabletAddRowsPayload& rows, @@ -46,8 +49,6 @@ class CloudDeltaWriter final : public BaseDeltaWriter { void update_tablet_stats(); - const RowsetMetaSharedPtr& rowset_meta(); - bool is_init() const { return _is_init; } static Status batch_init(std::vector writers); @@ -58,14 +59,12 @@ class CloudDeltaWriter final : public BaseDeltaWriter { std::shared_ptr resource_context() { return _resource_ctx; } private: - // Convert `_rowset_builder` from `BaseRowsetBuilder` to `CloudRowsetBuilder` CloudRowsetBuilder* rowset_builder(); // Handle commit for empty rowset (when no data is written) Status _commit_empty_rowset(); bthread::Mutex _mtx; - CloudStorageEngine& _engine; std::shared_ptr _resource_ctx; }; diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index 869edca0b4912a..e4c815c913bcaf 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -158,6 +158,12 @@ Status CloudTabletCalcDeleteBitmapTask::handle(int64_t queue_time_us) const { return Status::Error( "can't get tablet when calculate delete bitmap. tablet_id={}", _tablet_id); } + if (tablet->is_row_binlog_tablet()) { + VLOG_DEBUG << "skip calculating delete bitmap for row binlog tablet, tablet_id=" + << _tablet_id << ", txn_id=" << _transaction_id + << ", it will be handled with its base tablet"; + return Status::OK(); + } // After https://github.com/apache/doris/pull/50417, there may be multiple calc delete bitmap tasks // with different signatures on the same (txn_id, tablet_id) load in same BE. We use _rowset_update_lock // to avoid them being executed concurrently to avoid correctness problem. @@ -309,9 +315,10 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( std::shared_ptr publish_status; int64_t txn_expiration; TxnPublishInfo previous_publish_info; + RowBinlogTxnInfo attach_row_binlog; Status status = _engine.txn_delete_bitmap_cache().get_tablet_txn_info( transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration, - &partial_update_info, &publish_status, &previous_publish_info); + &partial_update_info, &publish_status, &previous_publish_info, &attach_row_binlog); if (status != Status::OK()) { LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id << ", " << txn_str << ", status=" << status; @@ -322,6 +329,7 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( TabletTxnInfo txn_info; txn_info.rowset = rowset; txn_info.delete_bitmap = delete_bitmap; + txn_info.attach_row_binlog = attach_row_binlog; txn_info.rowset_ids = rowset_ids; txn_info.partial_update_info = partial_update_info; txn_info.publish_status = publish_status; diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index a582f83991e7bf..e9bcbf8379567c 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -371,6 +371,9 @@ static std::string debug_info(const Request& req) { return ""; } else if constexpr (is_any_v) { return fmt::format(" tablet_id={}", req.rowset_meta().tablet_id()); + } else if constexpr (is_any_v) { + return fmt::format(" tablet_id={}, attach_row_binlog_tablet_id={}", + req.rowset_meta().tablet_id(), req.attach_row_binlog().tablet_id()); } else if constexpr (is_any_v) { return fmt::format(" tablet_id={}", req.tablet_id()); } else if constexpr (is_any_v) { @@ -801,9 +804,8 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, sync_stats->get_remote_rowsets_num += resp.rowset_meta().size(); } - // If is mow, the tablet has no delete bitmap in base rowsets. - // So dont need to sync it. - if (options.sync_delete_bitmap && tablet->enable_unique_key_merge_on_write() && + // MOW and row-binlog tablets need delete bitmap from meta-service. + if (options.sync_delete_bitmap && tablet->need_read_delete_bitmap() && tablet->tablet_state() == TABLET_RUNNING) { DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.sync_tablet_delete_bitmap.block", DBUG_BLOCK); @@ -1530,6 +1532,69 @@ Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_i return st; } +Status CloudMetaMgr::commit_rowsets(RowsetMeta& rs_meta, RowsetMeta& attach_row_binlog, + const std::string& job_id, int64_t table_id, + RowsetMetaSharedPtr* existed_rs_meta, + RowsetMetaSharedPtr* existed_attach_row_binlog) { + VLOG_DEBUG << "commit rowsets, tablet_id: " << rs_meta.tablet_id() + << ", rowset_id: " << rs_meta.rowset_id() + << ", attach_row_binlog_tablet_id: " << attach_row_binlog.tablet_id() + << ", attach_row_binlog_rowset_id: " << attach_row_binlog.rowset_id() + << " txn_id: " << rs_meta.txn_id(); + check_table_size_correctness(rs_meta); + check_table_size_correctness(attach_row_binlog); + + CreateRowsetsRequest req; + CreateRowsetsResponse resp; + req.set_cloud_unique_id(config::cloud_unique_id); + req.set_txn_id(rs_meta.txn_id()); + req.set_tablet_job_id(job_id); + + RowsetMetaPB rs_meta_pb = rs_meta.get_rowset_pb(); + doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(rs_meta_pb)); + RowsetMetaPB attach_row_binlog_pb = attach_row_binlog.get_rowset_pb(); + doris_rowset_meta_to_cloud(req.mutable_attach_row_binlog(), std::move(attach_row_binlog_pb)); + + Status st = + retry_rpc(MetaServiceRPC::COMMIT_ROWSET, req, &resp, &MetaService_Stub::commit_rowsets, + { + .host_limiters = host_level_ms_rpc_rate_limiters_, + .backpressure_handler = ms_backpressure_handler_, + .table_id = table_id, + }); + if (!st.ok() && resp.status().code() == MetaServiceCode::ALREADY_EXISTED) { + if (existed_rs_meta != nullptr && resp.has_existed_rowset_meta()) { + RowsetMetaPB doris_rs_meta = + cloud_rowset_meta_to_doris(std::move(*resp.mutable_existed_rowset_meta())); + *existed_rs_meta = std::make_shared(); + (*existed_rs_meta)->init_from_pb(doris_rs_meta); + } + if (existed_attach_row_binlog != nullptr && resp.has_existed_attach_row_binlog()) { + RowsetMetaPB doris_attach_row_binlog = cloud_rowset_meta_to_doris( + std::move(*resp.mutable_existed_attach_row_binlog())); + *existed_attach_row_binlog = std::make_shared(); + (*existed_attach_row_binlog)->init_from_pb(doris_attach_row_binlog); + } + return Status::AlreadyExist("failed to commit rowsets: {}", resp.status().msg()); + } + int64_t timeout_ms = -1; + if (config::enable_compaction_delay_commit_for_warm_up && !job_id.empty()) { + const double speed_mbps = 100.0; // 100MB/s + const double safety_factor = 2.0; + timeout_ms = std::min( + std::max(static_cast(static_cast(rs_meta.total_disk_size()) / + (speed_mbps * 1024 * 1024) * safety_factor * 1000), + config::warm_up_rowset_sync_wait_min_timeout_ms), + config::warm_up_rowset_sync_wait_max_timeout_ms); + LOG(INFO) << "warm up rowsets: " << rs_meta.version() << ", job_id: " << job_id + << ", with timeout: " << timeout_ms << " ms"; + } + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.warm_up_rowset(rs_meta, table_id, timeout_ms); + manager.warm_up_rowset(attach_row_binlog, table_id, timeout_ms); + return st; +} + void CloudMetaMgr::cache_committed_rowset(RowsetMetaSharedPtr rs_meta, int64_t expiration_time) { // For load-generated rowsets (job_id is empty), add to pending rowset manager // so FE can notify BE to promote them later @@ -1568,6 +1633,37 @@ Status CloudMetaMgr::update_tmp_rowset(const RowsetMeta& rs_meta, int64_t table_ return st; } +Status CloudMetaMgr::update_tmp_rowsets(const RowsetMeta& rs_meta, + const RowsetMeta& attach_row_binlog, int64_t table_id) { + VLOG_DEBUG << "update committed rowsets, tablet_id: " << rs_meta.tablet_id() + << ", rowset_id: " << rs_meta.rowset_id() + << ", attach_row_binlog_tablet_id: " << attach_row_binlog.tablet_id() + << ", attach_row_binlog_rowset_id: " << attach_row_binlog.rowset_id(); + CreateRowsetsRequest req; + CreateRowsetsResponse resp; + req.set_cloud_unique_id(config::cloud_unique_id); + + DCHECK_EQ(rs_meta.tablet_schema()->num_variant_columns(), + attach_row_binlog.tablet_schema()->num_variant_columns()); + bool skip_schema = rs_meta.tablet_schema()->num_variant_columns() == 0; + RowsetMetaPB rs_meta_pb = rs_meta.get_rowset_pb(skip_schema); + doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(rs_meta_pb)); + RowsetMetaPB attach_row_binlog_pb = attach_row_binlog.get_rowset_pb(skip_schema); + doris_rowset_meta_to_cloud(req.mutable_attach_row_binlog(), std::move(attach_row_binlog_pb)); + + Status st = retry_rpc(MetaServiceRPC::UPDATE_TMP_ROWSET, req, &resp, + &MetaService_Stub::update_tmp_rowsets, + { + .host_limiters = host_level_ms_rpc_rate_limiters_, + .backpressure_handler = ms_backpressure_handler_, + .table_id = table_id, + }); + if (!st.ok() && resp.status().code() == MetaServiceCode::ROWSET_META_NOT_FOUND) { + return Status::InternalError("failed to update committed rowsets: {}", resp.status().msg()); + } + return st; +} + // async send TableStats(in res) to FE coz we are in streamload ctx, response to the user ASAP static void send_stats_to_fe_async(const int64_t db_id, const int64_t txn_id, const std::string& label, CommitTxnResponse& res, diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 657265594bcd62..f685e6828ccc55 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -84,9 +84,16 @@ class CloudMetaMgr { Status commit_rowset(RowsetMeta& rs_meta, const std::string& job_id, int64_t table_id, std::shared_ptr* existed_rs_meta = nullptr); + + Status commit_rowsets(RowsetMeta& rs_meta, RowsetMeta& attach_row_binlog, + const std::string& job_id, int64_t table_id, + std::shared_ptr* existed_rs_meta = nullptr, + std::shared_ptr* existed_attach_row_binlog = nullptr); void cache_committed_rowset(RowsetMetaSharedPtr rs_meta, int64_t expiration_time); Status update_tmp_rowset(const RowsetMeta& rs_meta, int64_t table_id); + Status update_tmp_rowsets(const RowsetMeta& rs_meta, const RowsetMeta& attach_row_binlog, + int64_t table_id); Status update_packed_file_info(const std::string& packed_file_path, const cloud::PackedFileInfoPB& packed_file_info, diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index abe36ed5790d71..ac8db138784498 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -21,7 +21,11 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "storage/rowset/group_rowset_writer.h" +#include "storage/rowset/rowset_factory.h" +#include "storage/rowset/rowset_writer_context.h" #include "storage/storage_policy.h" +#include "storage/tablet_info.h" namespace doris { using namespace ErrorCode; @@ -30,6 +34,19 @@ CloudRowsetBuilder::CloudRowsetBuilder(CloudStorageEngine& engine, const WriteRe RuntimeProfile* profile) : BaseRowsetBuilder(req, profile), _engine(engine) {} +CloudGroupRowsetBuilder::CloudGroupRowsetBuilder(CloudStorageEngine& engine, + const WriteRequest& group_build_req, + const WriteRequest& sub_data_req, + const WriteRequest& sub_row_binlog_req, + RuntimeProfile* profile) + : CloudRowsetBuilder(engine, group_build_req, profile) { + DCHECK(group_build_req.write_req_type == WriteRequestType::GROUP && + sub_data_req.write_req_type == WriteRequestType::DATA && + sub_row_binlog_req.write_req_type == WriteRequestType::ROW_BINLOG); + _data_builder = std::make_shared(engine, sub_data_req, profile); + _row_binlog_builder = std::make_shared(engine, sub_row_binlog_req, profile); +} + CloudRowsetBuilder::~CloudRowsetBuilder() { // Clear file cache immediately when load fails if (_is_init && _rowset != nullptr && _rowset->rowset_meta()->rowset_state() == PREPARED) { @@ -41,7 +58,7 @@ Status CloudRowsetBuilder::init() { _tablet = DORIS_TRY(_engine.get_tablet(_req.tablet_id)); std::shared_ptr mow_context; - if (_tablet->enable_unique_key_merge_on_write()) { + if (_tablet->enable_unique_key_merge_on_write() && is_data_builder()) { if (config::cloud_mow_sync_rowsets_when_load_txn_begin) { auto st = std::static_pointer_cast(_tablet)->sync_rowsets(); // sync_rowsets will return INVALID_TABLET_STATE when tablet is under alter @@ -50,6 +67,10 @@ Status CloudRowsetBuilder::init() { } } RETURN_IF_ERROR(init_mow_context(mow_context)); + } else if (_req.write_req_type == WriteRequestType::ROW_BINLOG) { + // Row binlog tablets use txn_delete_bitmap_cache for local make-visible. + // The real binlog delete bitmap is derived when the base tablet calculates delete bitmap. + _delete_bitmap = std::make_shared(_req.tablet_id); } RETURN_IF_ERROR(check_tablet_version_count()); @@ -65,17 +86,25 @@ Status CloudRowsetBuilder::init() { context.txn_id = _req.txn_id; context.txn_expiration = _req.txn_expiration; context.load_id = _req.load_id; + context.db_id = _req.table_schema_param->db_id(); + context.table_id = _req.table_schema_param->table_id(); context.rowset_state = PREPARED; context.segments_overlap = OVERLAPPING; context.tablet_schema = _tablet_schema; context.newest_write_timestamp = UnixSeconds(); context.tablet_id = _req.tablet_id; + context.tablet_schema_hash = _req.schema_hash; context.index_id = _req.index_id; context.tablet = _tablet; + context.enable_segcompaction = true; + if (_req.write_req_type == WriteRequestType::ROW_BINLOG || !_attach_rowset_ids.empty()) { + context.enable_segcompaction = false; + } context.write_type = DataWriteType::TYPE_DIRECT; context.mow_context = mow_context; context.write_file_cache = _req.write_file_cache; context.partial_update_info = _partial_update_info; + context.write_binlog_opt().enable = _req.write_req_type == WriteRequestType::ROW_BINLOG; context.file_cache_ttl_sec = _tablet->ttl_seconds(); context.storage_resource = _engine.get_storage_resource(_req.storage_vault_id); if (!context.storage_resource) { @@ -84,6 +113,7 @@ Status CloudRowsetBuilder::init() { } _rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false)); + _rowset_id = context.rowset_id; _calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor()->create_token(); @@ -96,6 +126,77 @@ Status CloudRowsetBuilder::init() { return Status::OK(); } +Status CloudGroupRowsetBuilder::init() { + RETURN_IF_ERROR(_row_binlog_builder->init()); + RETURN_IF_ERROR( + _data_builder->attach_pending_rs_guard_to_txn(_row_binlog_builder->rowset_id())); + RETURN_IF_ERROR(_data_builder->init()); + _tablet = _data_builder->tablet_sptr(); + + std::unique_ptr group_writer; + RETURN_IF_ERROR(RowsetFactory::create_empty_group_rowset_writer(&group_writer)); + group_writer->set_data_writer(_data_builder->rowset_writer()); + group_writer->set_row_binlog_writer(_row_binlog_builder->rowset_writer()); + + { + const auto& data_ctx = _data_builder->rowset_writer()->context(); + auto& binlog_ctx = + const_cast(_row_binlog_builder->rowset_writer()->context()); + auto& cfg = binlog_ctx.write_binlog_opt().write_binlog_config(); + cfg.source.tablet_schema = data_ctx.tablet_schema; + cfg.source.partial_update_info = data_ctx.partial_update_info; + cfg.source.mow_context = data_ctx.mow_context; + cfg.source.is_transient_rowset_writer = data_ctx.is_transient_rowset_writer; + cfg.source.source_write_type = data_ctx.write_type; + cfg.source.base_tablet = _data_builder->tablet_sptr(); + } + + _rowset_writer = std::move(group_writer); + _is_init = true; + return Status::OK(); +} + +Status CloudGroupRowsetBuilder::build_rowset() { + RETURN_IF_ERROR(_row_binlog_builder->build_rowset()); + return _data_builder->build_rowset(); +} + +Status CloudGroupRowsetBuilder::submit_calc_delete_bitmap_task() { + return _data_builder->submit_calc_delete_bitmap_task(); +} + +Status CloudGroupRowsetBuilder::wait_calc_delete_bitmap() { + return _data_builder->wait_calc_delete_bitmap(); +} + +void CloudGroupRowsetBuilder::update_tablet_stats() { + _data_builder->update_tablet_stats(); + _row_binlog_builder->update_tablet_stats(); +} + +Status CloudGroupRowsetBuilder::commit_rowset(const std::string& job_id, int64_t table_id) { + return _engine.meta_mgr().commit_rowsets(*_data_builder->rowset_meta(), + *_row_binlog_builder->rowset_meta(), job_id, table_id); +} + +Status CloudGroupRowsetBuilder::set_txn_related_info() { + RowBinlogTxnInfo attach_row_binlog; + attach_row_binlog.rowset = _row_binlog_builder->rowset(); + attach_row_binlog.tablet = _row_binlog_builder->tablet_sptr(); + if (_data_builder->tablet()->enable_unique_key_merge_on_write()) { + attach_row_binlog.delete_bitmap = + std::make_shared(_row_binlog_builder->tablet()->tablet_id()); + } + RETURN_IF_ERROR(_data_builder->attach_row_binlog_to_txn(attach_row_binlog)); + RETURN_IF_ERROR(_data_builder->set_txn_related_info()); + return _row_binlog_builder->set_txn_related_info(); +} + +void CloudGroupRowsetBuilder::set_skip_writing_rowset_metadata(bool skip) { + _data_builder->set_skip_writing_rowset_metadata(skip); + _row_binlog_builder->set_skip_writing_rowset_metadata(skip); +} + Status CloudRowsetBuilder::check_tablet_version_count() { int64_t version_count = cloud_tablet()->fetch_add_approximate_num_rowsets(0); DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version", @@ -134,8 +235,12 @@ const RowsetMetaSharedPtr& CloudRowsetBuilder::rowset_meta() { return _rowset_writer->rowset_meta(); } +Status CloudRowsetBuilder::commit_rowset(const std::string& job_id, int64_t table_id) { + return _engine.meta_mgr().commit_rowset(*rowset_meta(), job_id, table_id); +} + Status CloudRowsetBuilder::set_txn_related_info() { - if (_tablet->enable_unique_key_merge_on_write()) { + if (_tablet->enable_unique_key_merge_on_write() || _tablet->is_row_binlog_tablet()) { // For empty rowsets when skip_writing_empty_rowset_metadata=true, // store only a lightweight marker instead of full rowset info. // This allows CalcDeleteBitmapTask to detect and skip gracefully, @@ -145,7 +250,8 @@ Status CloudRowsetBuilder::set_txn_related_info() { _req.txn_expiration); return Status::OK(); } - if (config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0) { + if (config::enable_merge_on_write_correctness_check && + _tablet->enable_unique_key_merge_on_write() && _rowset->num_rows() != 0) { auto st = _tablet->check_delete_bitmap_correctness( _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids); if (!st.ok()) { @@ -159,9 +265,10 @@ Status CloudRowsetBuilder::set_txn_related_info() { } _engine.txn_delete_bitmap_cache().set_tablet_txn_info( _req.txn_id, _tablet->tablet_id(), _delete_bitmap, *_rowset_ids, _rowset, - _req.txn_expiration, _partial_update_info); + _req.txn_expiration, _partial_update_info, _attach_row_binlog); } else { - if (config::enable_cloud_make_rs_visible_on_be) { + // TSO-enabled rowsets must become visible from MS rowset meta. + if (config::enable_cloud_make_rs_visible_on_be && !_tablet_schema->enable_tso()) { if (_skip_writing_rowset_metadata) { _engine.committed_rs_mgr().mark_empty_rowset(_req.txn_id, _tablet->tablet_id(), _req.txn_expiration); diff --git a/be/src/cloud/cloud_rowset_builder.h b/be/src/cloud/cloud_rowset_builder.h index cec8cfed979857..b4a9ead311e6ce 100644 --- a/be/src/cloud/cloud_rowset_builder.h +++ b/be/src/cloud/cloud_rowset_builder.h @@ -24,7 +24,7 @@ namespace doris { class CloudTablet; class CloudStorageEngine; -class CloudRowsetBuilder final : public BaseRowsetBuilder { +class CloudRowsetBuilder : public BaseRowsetBuilder { public: CloudRowsetBuilder(CloudStorageEngine& engine, const WriteRequest& req, RuntimeProfile* profile); @@ -33,15 +33,19 @@ class CloudRowsetBuilder final : public BaseRowsetBuilder { Status init() override; - void update_tablet_stats(); + virtual void update_tablet_stats(); const RowsetMetaSharedPtr& rowset_meta(); - Status set_txn_related_info(); + virtual Status commit_rowset(const std::string& job_id, int64_t table_id); - void set_skip_writing_rowset_metadata(bool skip) { _skip_writing_rowset_metadata = skip; } + virtual Status set_txn_related_info(); -private: + virtual void set_skip_writing_rowset_metadata(bool skip) { + _skip_writing_rowset_metadata = skip; + } + +protected: // Convert `_tablet` from `BaseTablet` to `CloudTablet` CloudTablet* cloud_tablet(); @@ -54,4 +58,47 @@ class CloudRowsetBuilder final : public BaseRowsetBuilder { bool _skip_writing_rowset_metadata = false; }; +class CloudGroupRowsetBuilder final : public CloudRowsetBuilder { +public: + CloudGroupRowsetBuilder(CloudStorageEngine& engine, const WriteRequest& group_build_req, + const WriteRequest& sub_data_req, + const WriteRequest& sub_row_binlog_req, RuntimeProfile* profile); + + Status init() override; + + Status build_rowset() override; + + Status submit_calc_delete_bitmap_task() override; + + Status wait_calc_delete_bitmap() override; + + void update_tablet_stats() override; + + Status commit_rowset(const std::string& job_id, int64_t table_id) override; + + Status set_txn_related_info() override; + + void set_skip_writing_rowset_metadata(bool skip) override; + + const BaseTabletSPtr& tablet_sptr() const override { return _data_builder->tablet_sptr(); } + + const RowsetSharedPtr& rowset() const override { return _data_builder->rowset(); } + + const TabletSchemaSPtr& tablet_schema() const override { + return _data_builder->tablet_schema(); + } + + const std::shared_ptr& get_partial_update_info() const override { + return _data_builder->get_partial_update_info(); + } + + CloudRowsetBuilder* data_builder() { return _data_builder.get(); } + + CloudRowsetBuilder* row_binlog_builder() { return _row_binlog_builder.get(); } + +private: + std::shared_ptr _data_builder; + std::shared_ptr _row_binlog_builder; +}; + } // namespace doris diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index 81645c791e29c9..6dd107a1131821 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -75,6 +75,9 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context) } _rowset_meta->set_tablet_schema(_context.tablet_schema); _rowset_meta->set_job_id(_context.job_id); + if (_context.write_binlog_opt().enable) { + _rowset_meta->mark_row_binlog(); + } _context.segment_collector = std::make_shared>(this); _context.file_writer_creator = std::make_shared>(this); if (_context.mow_context != nullptr) { diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 0dc8321c2c4e3a..c4c6f636644f0c 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -33,6 +33,7 @@ #include "cloud/cloud_base_compaction.h" #include "cloud/cloud_compaction_stop_token.h" #include "cloud/cloud_cumulative_compaction.h" +#include "cloud/cloud_cumulative_compaction_binlog_policy.h" #include "cloud/cloud_cumulative_compaction_policy.h" #include "cloud/cloud_full_compaction.h" #include "cloud/cloud_index_change_compaction.h" @@ -59,6 +60,7 @@ #include "load/memtable/memtable_flush_executor.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction_task_tracker.h" @@ -93,6 +95,15 @@ int get_base_thread_num() { return std::min(std::max(int(num_cores * config::base_compaction_thread_num_factor), 1), 10); } +int get_binlog_thread_num() { + if (config::max_binlog_compaction_threads > 0) { + return config::max_binlog_compaction_threads; + } + + int num_cores = doris::CpuInfo::num_cores(); + return std::min(std::max(int(num_cores * config::binlog_compaction_thread_num_factor), 1), 10); +} + CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) : BaseStorageEngine(Type::CLOUD, options.backend_uid), _meta_mgr(std::make_unique()), @@ -102,6 +113,8 @@ CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) std::make_shared(); _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = std::make_shared(); + _cumulative_compaction_policies[CUMULATIVE_BINLOG_POLICY] = + std::make_shared(); _startup_timepoint = std::chrono::system_clock::now(); } @@ -260,6 +273,24 @@ Status CloudStorageEngine::open() { return Status::OK(); } +#ifdef BE_TEST +void CloudStorageEngine::init_calc_delete_bitmap_executor_for_UT() { + if (_calc_delete_bitmap_executor == nullptr) { + _calc_delete_bitmap_executor = std::make_unique(); + _calc_delete_bitmap_executor->init("TabletCalcDeleteBitmapThreadPool", + config::calc_delete_bitmap_max_thread); + } + if (_calc_delete_bitmap_executor_for_load == nullptr) { + _calc_delete_bitmap_executor_for_load = std::make_unique(); + _calc_delete_bitmap_executor_for_load->init( + "LoadCalcDeleteBitmapThreadPool", + config::calc_delete_bitmap_for_load_max_thread > 0 + ? config::calc_delete_bitmap_for_load_max_thread + : std::max(1, CpuInfo::num_cores() / 2)); + } +} +#endif + void CloudStorageEngine::stop() { if (_stopped) { return; @@ -280,6 +311,9 @@ void CloudStorageEngine::stop() { if (_cumu_compaction_thread_pool) { _cumu_compaction_thread_pool->shutdown(); } + if (_binlog_compaction_thread_pool) { + _binlog_compaction_thread_pool->shutdown(); + } _adaptive_thread_controller.stop(); LOG(INFO) << "Cloud storage engine is stopped."; @@ -372,6 +406,7 @@ Status CloudStorageEngine::start_bg_threads(std::shared_ptr wg_sp // compaction tasks producer thread int base_thread_num = get_base_thread_num(); int cumu_thread_num = get_cumu_thread_num(); + int binlog_thread_num = get_binlog_thread_num(); RETURN_IF_ERROR(ThreadPoolBuilder("BaseCompactionTaskThreadPool") .set_min_threads(base_thread_num) @@ -381,12 +416,21 @@ Status CloudStorageEngine::start_bg_threads(std::shared_ptr wg_sp .set_min_threads(cumu_thread_num) .set_max_threads(cumu_thread_num) .build(&_cumu_compaction_thread_pool)); + RETURN_IF_ERROR(ThreadPoolBuilder("BinlogCompactionTaskThreadPool") + .set_min_threads(binlog_thread_num) + .set_max_threads(binlog_thread_num) + .build(&_binlog_compaction_thread_pool)); RETURN_IF_ERROR(Thread::create( "StorageEngine", "compaction_tasks_producer_thread", [this]() { this->_compaction_tasks_producer_callback(); }, &_bg_threads.emplace_back())); + RETURN_IF_ERROR(Thread::create( + "StorageEngine", "binlog_compaction_tasks_producer_thread", + [this]() { this->_binlog_compaction_tasks_producer_callback(); }, + &_bg_threads.emplace_back())); LOG(INFO) << "compaction tasks producer thread started," - << " base thread num " << base_thread_num << " cumu thread num " << cumu_thread_num; + << " base thread num " << base_thread_num << " cumu thread num " << cumu_thread_num + << " binlog thread num " << binlog_thread_num; RETURN_IF_ERROR(Thread::create( "StorageEngine", "lease_compaction_thread", @@ -491,8 +535,9 @@ void CloudStorageEngine::get_cumu_compaction( Status CloudStorageEngine::_adjust_compaction_thread_num() { int base_thread_num = get_base_thread_num(); - if (!_base_compaction_thread_pool || !_cumu_compaction_thread_pool) { - LOG(WARNING) << "base or cumu compaction thread pool is not created"; + if (!_base_compaction_thread_pool || !_cumu_compaction_thread_pool || + !_binlog_compaction_thread_pool) { + LOG(WARNING) << "compaction thread pool is not created"; return Status::Error(""); } @@ -530,6 +575,24 @@ Status CloudStorageEngine::_adjust_compaction_thread_num() { << " to " << cumu_thread_num; } } + + int binlog_thread_num = get_binlog_thread_num(); + if (_binlog_compaction_thread_pool->max_threads() != binlog_thread_num) { + int old_max_threads = _binlog_compaction_thread_pool->max_threads(); + Status status = _binlog_compaction_thread_pool->set_max_threads(binlog_thread_num); + if (status.ok()) { + VLOG_NOTICE << "update binlog compaction thread pool max_threads from " + << old_max_threads << " to " << binlog_thread_num; + } + } + if (_binlog_compaction_thread_pool->min_threads() != binlog_thread_num) { + int old_min_threads = _binlog_compaction_thread_pool->min_threads(); + Status status = _binlog_compaction_thread_pool->set_min_threads(binlog_thread_num); + if (status.ok()) { + VLOG_NOTICE << "update binlog compaction thread pool min_threads from " + << old_min_threads << " to " << binlog_thread_num; + } + } return Status::OK(); } @@ -615,6 +678,47 @@ void CloudStorageEngine::_compaction_tasks_producer_callback() { } while (!_stop_background_threads_latch.wait_for(std::chrono::milliseconds(interval))); } +void CloudStorageEngine::_binlog_compaction_tasks_producer_callback() { + LOG(INFO) << "try to start binlog compaction producer process!"; + + int64_t last_binlog_score_update_time = 0; + static const int64_t check_score_interval_ms = 5000; + + int64_t interval = config::generate_compaction_tasks_interval_ms; + do { + int64_t cur_time = UnixMillis(); + if (config::enable_feature_binlog && !config::disable_auto_compaction) { + Status st = _adjust_compaction_thread_num(); + if (!st.ok()) { + break; + } + + bool check_score = false; + if (cur_time - last_binlog_score_update_time >= check_score_interval_ms) { + check_score = true; + last_binlog_score_update_time = cur_time; + } + + std::vector tablets_compaction = _generate_cloud_compaction_tasks( + CompactionType::CUMU_BINLOG_COMPACTION, check_score); + for (const auto& tablet : tablets_compaction) { + Status status = + submit_compaction_task(tablet, CompactionType::CUMU_BINLOG_COMPACTION); + if (status.ok()) continue; + if ((!status.is() && + !status.is()) || + VLOG_DEBUG_IS_ON) { + LOG(WARNING) << "failed to submit binlog compaction task for tablet: " + << tablet->tablet_id() << ", err: " << status; + } + } + interval = config::generate_compaction_tasks_interval_ms; + } else { + interval = config::check_auto_compaction_interval_seconds * 1000; + } + } while (!_stop_background_threads_latch.wait_for(std::chrono::milliseconds(interval))); +} + void CloudStorageEngine::unregister_index_change_compaction(int64_t tablet_id, bool is_base_compact) { std::lock_guard lock(_compaction_mtx); @@ -685,14 +789,17 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task } bool need_pick_tablet = true; - int thread_per_disk = - config::compaction_task_num_per_fast_disk; // all disks are fast in cloud mode + int thread_per_disk = compaction_type == CompactionType::CUMU_BINLOG_COMPACTION + ? config::binlog_compaction_task_num_per_disk + : config::compaction_task_num_per_fast_disk; int num_cumu = std::accumulate(submitted_cumu_compactions.begin(), submitted_cumu_compactions.end(), 0, [](int a, auto& b) { return a + b.second.size(); }); int num_base = cast_set(submitted_base_compactions.size() + submitted_full_compactions.size()); - int n = thread_per_disk - num_cumu - num_base; + int n = compaction_type == CompactionType::CUMU_BINLOG_COMPACTION + ? thread_per_disk + : thread_per_disk - num_cumu - num_base; if (compaction_type == CompactionType::BASE_COMPACTION) { // We need to reserve at least one thread for cumulative compaction, // because base compactions may take too long to complete, which may @@ -712,15 +819,27 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task if (compaction_type == CompactionType::BASE_COMPACTION) { filter_out = [&submitted_base_compactions, &submitted_full_compactions, &submitted_index_change_base_compactions](CloudTablet* t) { - return submitted_base_compactions.contains(t->tablet_id()) || + return t->is_row_binlog_tablet() || + submitted_base_compactions.contains(t->tablet_id()) || submitted_full_compactions.contains(t->tablet_id()) || submitted_index_change_base_compactions.contains(t->tablet_id()) || t->tablet_state() != TABLET_RUNNING; }; + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + filter_out = [&tablet_preparing_cumu_compaction, &submitted_cumu_compactions, + &submitted_index_change_cumu_compactions](CloudTablet* t) { + return !t->is_row_binlog_tablet() || + tablet_preparing_cumu_compaction.contains(t->tablet_id()) || + submitted_index_change_cumu_compactions.contains(t->tablet_id()) || + submitted_cumu_compactions.contains(t->tablet_id()) || + (t->tablet_state() != TABLET_RUNNING && + (!config::enable_new_tablet_do_compaction || t->alter_version() == -1)); + }; } else if (config::enable_parallel_cumu_compaction) { filter_out = [&tablet_preparing_cumu_compaction, &submitted_index_change_cumu_compactions](CloudTablet* t) { - return tablet_preparing_cumu_compaction.contains(t->tablet_id()) || + return t->is_row_binlog_tablet() || + tablet_preparing_cumu_compaction.contains(t->tablet_id()) || submitted_index_change_cumu_compactions.contains(t->tablet_id()) || (t->tablet_state() != TABLET_RUNNING && (!config::enable_new_tablet_do_compaction || t->alter_version() == -1)); @@ -728,7 +847,8 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task } else { filter_out = [&tablet_preparing_cumu_compaction, &submitted_cumu_compactions, &submitted_index_change_cumu_compactions](CloudTablet* t) { - return tablet_preparing_cumu_compaction.contains(t->tablet_id()) || + return t->is_row_binlog_tablet() || + tablet_preparing_cumu_compaction.contains(t->tablet_id()) || submitted_index_change_cumu_compactions.contains(t->tablet_id()) || submitted_cumu_compactions.contains(t->tablet_id()) || (t->tablet_state() != TABLET_RUNNING && @@ -754,6 +874,9 @@ std::vector CloudStorageEngine::_generate_cloud_compaction_task if (compaction_type == CompactionType::BASE_COMPACTION) { DorisMetrics::instance()->tablet_base_max_compaction_score->set_value( max_compaction_score); + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + DorisMetrics::instance()->tablet_binlog_max_compaction_score->set_value( + max_compaction_score); } else { DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value( max_compaction_score); @@ -920,7 +1043,10 @@ Status CloudStorageEngine::_submit_base_compaction_task(const CloudTabletSPtr& t } Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletSPtr& tablet, - int trigger_method) { + int trigger_method, + CompactionType compaction_type) { + DCHECK(compaction_type == CompactionType::CUMULATIVE_COMPACTION || + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION); using namespace std::chrono; { std::lock_guard lock(_compaction_mtx); @@ -1018,10 +1144,19 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS tablet->last_cumu_no_suitable_version_ms = 0; } }; - st = _cumu_compaction_thread_pool->submit_func([=, this, compaction = std::move(compaction)]() { - DorisMetrics::instance()->cumulative_compaction_task_running_total->increment(1); - DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( - _cumu_compaction_thread_pool->get_queue_size()); + auto& submit_thread_pool = compaction_type == CompactionType::CUMU_BINLOG_COMPACTION + ? _binlog_compaction_thread_pool + : _cumu_compaction_thread_pool; + st = submit_thread_pool->submit_func([=, this, compaction = std::move(compaction)]() { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + DorisMetrics::instance()->binlog_compaction_task_running_total->increment(1); + DorisMetrics::instance()->binlog_compaction_task_pending_total->set_value( + _binlog_compaction_thread_pool->get_queue_size()); + } else { + DorisMetrics::instance()->cumulative_compaction_task_running_total->increment(1); + DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( + _cumu_compaction_thread_pool->get_queue_size()); + } DBUG_EXECUTE_IF("CloudStorageEngine._submit_cumulative_compaction_task.wait_in_line", { sleep(5); }) signal::tablet_id = tablet->tablet_id(); @@ -1032,16 +1167,24 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS { sleep(5); }) // Idempotent cleanup: remove task from tracker CompactionTaskTracker::instance()->remove_task(compaction_id); - std::lock_guard lock(_cumu_compaction_delay_mtx); - _cumu_compaction_thread_pool_used_threads--; - if (!is_large_task) { - _cumu_compaction_thread_pool_small_tasks_running--; + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + std::lock_guard lock(_cumu_compaction_delay_mtx); + _cumu_compaction_thread_pool_used_threads--; + if (!is_large_task) { + _cumu_compaction_thread_pool_small_tasks_running--; + } } g_cumu_compaction_running_task_count << -1; erase_submitted_cumu_compaction(); - DorisMetrics::instance()->cumulative_compaction_task_running_total->increment(-1); - DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( - _cumu_compaction_thread_pool->get_queue_size()); + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + DorisMetrics::instance()->binlog_compaction_task_running_total->increment(-1); + DorisMetrics::instance()->binlog_compaction_task_pending_total->set_value( + _binlog_compaction_thread_pool->get_queue_size()); + } else { + DorisMetrics::instance()->cumulative_compaction_task_running_total->increment(-1); + DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( + _cumu_compaction_thread_pool->get_queue_size()); + } }}; auto st = _request_tablet_global_compaction_lock(ReaderType::READER_CUMULATIVE_COMPACTION, tablet, compaction); @@ -1054,6 +1197,9 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS CompactionTaskTracker::instance()->update_to_running(compaction_id, rs); } do { + if (compaction_type != CompactionType::CUMULATIVE_COMPACTION) { + break; + } std::lock_guard lock(_cumu_compaction_delay_mtx); _cumu_compaction_thread_pool_used_threads++; if (config::large_cumu_compaction_task_min_thread_num > 1 && @@ -1101,8 +1247,13 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS } erase_executing_cumu_compaction(); }); - DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( - _cumu_compaction_thread_pool->get_queue_size()); + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + DorisMetrics::instance()->binlog_compaction_task_pending_total->set_value( + _binlog_compaction_thread_pool->get_queue_size()); + } else { + DorisMetrics::instance()->cumulative_compaction_task_pending_total->set_value( + _cumu_compaction_thread_pool->get_queue_size()); + } if (!st.ok()) { tracker->remove_task(compaction_id); erase_submitted_cumu_compaction(); @@ -1112,6 +1263,12 @@ Status CloudStorageEngine::_submit_cumulative_compaction_task(const CloudTabletS return st; } +Status CloudStorageEngine::_submit_binlog_compaction_task(const CloudTabletSPtr& tablet, + int trigger_method) { + return _submit_cumulative_compaction_task(tablet, trigger_method, + CompactionType::CUMU_BINLOG_COMPACTION); +} + Status CloudStorageEngine::_submit_full_compaction_task(const CloudTabletSPtr& tablet, int trigger_method) { using namespace std::chrono; @@ -1209,6 +1366,7 @@ Status CloudStorageEngine::submit_compaction_task(const CloudTabletSPtr& tablet, int trigger_method) { DCHECK(compaction_type == CompactionType::CUMULATIVE_COMPACTION || compaction_type == CompactionType::BASE_COMPACTION || + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION || compaction_type == CompactionType::FULL_COMPACTION); switch (compaction_type) { case CompactionType::BASE_COMPACTION: @@ -1217,6 +1375,9 @@ Status CloudStorageEngine::submit_compaction_task(const CloudTabletSPtr& tablet, case CompactionType::CUMULATIVE_COMPACTION: RETURN_IF_ERROR(_submit_cumulative_compaction_task(tablet, trigger_method)); return Status::OK(); + case CompactionType::CUMU_BINLOG_COMPACTION: + RETURN_IF_ERROR(_submit_binlog_compaction_task(tablet, trigger_method)); + return Status::OK(); case CompactionType::FULL_COMPACTION: RETURN_IF_ERROR(_submit_full_compaction_task(tablet, trigger_method)); return Status::OK(); diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index b939bbf094a1d5..87b8e3ba2630ed 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -198,6 +198,7 @@ class CloudStorageEngine final : public BaseStorageEngine { } void set_cloud_warm_up_manager(std::unique_ptr manager); + void init_calc_delete_bitmap_executor_for_UT(); #endif private: @@ -205,12 +206,15 @@ class CloudStorageEngine final : public BaseStorageEngine { void _vacuum_stale_rowsets_thread_callback(); void _sync_tablets_thread_callback(); void _compaction_tasks_producer_callback(); + void _binlog_compaction_tasks_producer_callback(); std::vector _generate_cloud_compaction_tasks(CompactionType compaction_type, bool check_score); Status _adjust_compaction_thread_num(); Status _submit_base_compaction_task(const CloudTabletSPtr& tablet, int trigger_method = 0); - Status _submit_cumulative_compaction_task(const CloudTabletSPtr& tablet, - int trigger_method = 0); + Status _submit_cumulative_compaction_task( + const CloudTabletSPtr& tablet, int trigger_method = 0, + CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION); + Status _submit_binlog_compaction_task(const CloudTabletSPtr& tablet, int trigger_method = 0); Status _submit_full_compaction_task(const CloudTabletSPtr& tablet, int trigger_method = 0); Status _request_tablet_global_compaction_lock(ReaderType compaction_type, const CloudTabletSPtr& tablet, diff --git a/be/src/cloud/cloud_stream_load_executor.cpp b/be/src/cloud/cloud_stream_load_executor.cpp index a92d2285bf25e3..c57a02c5976408 100644 --- a/be/src/cloud/cloud_stream_load_executor.cpp +++ b/be/src/cloud/cloud_stream_load_executor.cpp @@ -67,7 +67,7 @@ Status CloudStreamLoadExecutor::operate_txn_2pc(StreamLoadContext* ctx) { Status st = Status::InternalError("impossible branch reached, " + op_info); if (ctx->txn_operation.compare("commit") == 0) { - if (!config::enable_stream_load_commit_txn_on_be) { + if (ctx->enable_tso() || !config::enable_stream_load_commit_txn_on_be) { VLOG_DEBUG << "2pc commit stream load txn with FE support: " << op_info; st = StreamLoadExecutor::operate_txn_2pc(ctx); } else if (topt == TxnOpParamType::WITH_TXN_ID) { @@ -109,8 +109,8 @@ Status CloudStreamLoadExecutor::commit_txn(StreamLoadContext* ctx) { volatile int* p = nullptr; *p = 1; }); - // forward to fe to excute commit transaction for MoW table - if (ctx->is_mow_table() || !config::enable_stream_load_commit_txn_on_be || + // Forward to FE for tables that need commit ordering metadata. + if (ctx->is_mow_table() || ctx->enable_tso() || !config::enable_stream_load_commit_txn_on_be || ctx->load_type == TLoadType::ROUTINE_LOAD) { Status st; int retry_times = 0; diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 7827c438bf9f54..2fd35dc005c65b 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -814,6 +814,9 @@ void CloudTablet::reset_approximate_stats(int64_t num_rowsets, int64_t num_segme auto cp = _cumulative_point.load(std::memory_order_relaxed); for (auto& [v, r] : _rs_version_map) { if (v.second < cp) { + if (is_row_binlog_tablet()) { + cumu_num_deltas += 1; + } continue; } cumu_num_deltas += r->is_segments_overlapping() ? r->num_segments() : 1; @@ -839,6 +842,10 @@ Result> CloudTablet::create_rowset_writer( context.partition_id = partition_id(); context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write(); context.encrypt_algorithm = tablet_meta()->encryption_algorithm(); + if (context.write_binlog_opt().enable) { + context.write_binlog_opt().set_need_before( + tablet_meta()->binlog_config().need_historical_value()); + } return RowsetFactory::create_rowset_writer(_engine, context, vertical); } @@ -863,6 +870,8 @@ Result> CloudTablet::create_transient_rowset_write RowsetWriterContext context; context.rowset_state = PREPARED; context.segments_overlap = OVERLAPPING; + context.db_id = rowset.rowset_meta()->db_id(); + context.table_id = rowset.rowset_meta()->table_id(); // During a partial update, the extracted columns of a variant should not be included in the tablet schema. // This is because the partial update for a variant needs to ignore the extracted columns. // Otherwise, the schema types in different rowsets might be inconsistent. When performing a partial update, @@ -874,6 +883,11 @@ Result> CloudTablet::create_transient_rowset_write context.write_type = DataWriteType::TYPE_DIRECT; context.partial_update_info = std::move(partial_update_info); context.is_transient_rowset_writer = true; + if (rowset.rowset_meta() != nullptr && rowset.rowset_meta()->is_row_binlog()) { + context.write_binlog_opt().enable = true; + context.write_binlog_opt().set_need_before( + tablet_meta()->binlog_config().need_historical_value()); + } context.rowset_id = rowset.rowset_id(); context.tablet_id = tablet_id(); context.index_id = index_id(); @@ -1197,21 +1211,55 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t tx int64_t next_visible_version) { RowsetSharedPtr rowset = txn_info->rowset; int64_t cur_version = rowset->start_version(); + const bool build_row_binlog = txn_info->attach_row_binlog.rowset != nullptr; + const RowsetId cur_build_rid = rowset->rowset_id(); + const RowsetId binlog_rid = + build_row_binlog ? txn_info->attach_row_binlog.rowset->rowset_id() : cur_build_rid; + auto* row_binlog_delete_bitmap = txn_info->attach_row_binlog.delete_bitmap.get(); + if (build_row_binlog) { + DCHECK(txn_info->attach_row_binlog.rowset->rowset_meta() != nullptr); + DCHECK(txn_info->attach_row_binlog.rowset->rowset_meta()->is_row_binlog()); + DCHECK(txn_info->attach_row_binlog.tablet != nullptr); + DCHECK(row_binlog_delete_bitmap != nullptr); + } + // update delete bitmap info, in order to avoid recalculation when trying again RETURN_IF_ERROR(_engine.txn_delete_bitmap_cache().update_tablet_txn_info( txn_id, tablet_id(), delete_bitmap, cur_rowset_ids, PublishStatus::PREPARE)); - if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update() && + if (txn_info->partial_update_info != nullptr && rowset_writer != nullptr && rowset_writer->num_rows() > 0) { DBUG_EXECUTE_IF("CloudTablet::save_delete_bitmap.update_tmp_rowset.error", { return Status::InternalError("injected update_tmp_rowset error."); }); const auto& rowset_meta = rowset->rowset_meta(); - RETURN_IF_ERROR(_engine.meta_mgr().update_tmp_rowset(*rowset_meta, table_id())); + if (build_row_binlog) { + const auto& binlog_rowset_meta = txn_info->attach_row_binlog.rowset->rowset_meta(); + RETURN_IF_ERROR(_engine.meta_mgr().update_tmp_rowsets(*rowset_meta, *binlog_rowset_meta, + table_id())); + } else { + RETURN_IF_ERROR(_engine.meta_mgr().update_tmp_rowset(*rowset_meta, table_id())); + } } RETURN_IF_ERROR(save_delete_bitmap_to_ms(cur_version, txn_id, delete_bitmap, lock_id, next_visible_version, rowset)); + if (build_row_binlog) { + for (const auto& [key, bitmap] : delete_bitmap->delete_bitmap) { + if (std::get<1>(key) != DeleteBitmap::INVALID_SEGMENT_ID && + std::get<0>(key) == cur_build_rid) { + row_binlog_delete_bitmap->merge( + {binlog_rid, std::get<1>(key), cast_set(cur_version)}, bitmap); + } + } + auto* binlog_tablet = static_cast(txn_info->attach_row_binlog.tablet.get()); + RETURN_IF_ERROR(binlog_tablet->save_delete_bitmap_to_ms( + cur_version, txn_id, txn_info->attach_row_binlog.delete_bitmap, lock_id, + next_visible_version, txn_info->attach_row_binlog.rowset)); + RETURN_IF_ERROR(_engine.txn_delete_bitmap_cache().update_tablet_txn_info( + txn_id, binlog_tablet->tablet_id(), txn_info->attach_row_binlog.delete_bitmap, + cur_rowset_ids, PublishStatus::SUCCEED, txn_info->publish_info)); + } // store the delete bitmap with sentinel marks in txn_delete_bitmap_cache because if the txn is retried for some reason, // it will use the delete bitmap from txn_delete_bitmap_cache when re-calculating the delete bitmap, during which it will do @@ -1970,6 +2018,10 @@ void CloudTablet::clear_unused_visible_pending_rowsets() { void CloudTablet::try_make_committed_rs_visible(int64_t txn_id, int64_t visible_version, int64_t version_update_time_ms) { + if (tablet_schema()->enable_tso()) { + return; + } + if (enable_unique_key_merge_on_write()) { // for mow tablet, we get committed rowset from `CloudTxnDeleteBitmapCache` rather than `CommittedRowsetManager` try_make_committed_rs_visible_for_mow(txn_id, visible_version, version_update_time_ms); diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 7fc3dec14512d2..5a3df79b38579d 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -488,7 +488,6 @@ class CloudTablet final : public BaseTablet { std::atomic _last_base_compaction_schedule_millis; // timestamp of last full compaction schedule time std::atomic _last_full_compaction_schedule_millis; - std::string _last_cumu_compaction_status; std::string _last_base_compaction_status; std::string _last_full_compaction_status; diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 3e979864138645..4461386fd3b839 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -188,6 +188,12 @@ Result> CloudTabletMgr::get_tablet(int64_t tablet_i auto* handle = _cache->lookup(key); if (handle == nullptr) { +#ifdef BE_TEST + if (auto tablet = _tablet_map->get(tablet_id); tablet != nullptr) { + set_tablet_access_time_ms(tablet.get()); + return tablet; + } +#endif if (force_use_only_cached) { LOG(INFO) << "tablet=" << tablet_id << "does not exists in local tablet cache, because param " @@ -428,13 +434,21 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( int n, CompactionType compaction_type, const std::function& filter_out, std::vector>* tablets, int64_t* max_score) { DCHECK(compaction_type == CompactionType::BASE_COMPACTION || - compaction_type == CompactionType::CUMULATIVE_COMPACTION); + compaction_type == CompactionType::CUMULATIVE_COMPACTION || + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION); *max_score = 0; int64_t max_score_tablet_id = 0; // clang-format off auto score = [compaction_type](CloudTablet* t) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION && !t->is_row_binlog_tablet()) { + return int64_t {0}; + } + if (compaction_type != CompactionType::CUMU_BINLOG_COMPACTION && t->is_row_binlog_tablet()) { + return int64_t {0}; + } return compaction_type == CompactionType::BASE_COMPACTION ? t->get_cloud_base_compaction_score() - : compaction_type == CompactionType::CUMULATIVE_COMPACTION ? t->get_cloud_cumu_compaction_score() + : (compaction_type == CompactionType::CUMULATIVE_COMPACTION || + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) ? t->get_cloud_cumu_compaction_score() : 0; }; @@ -489,6 +503,7 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( int64_t s = score(t.get()); if (s <= 0) { continue; } + if (s > *max_score) { max_score_tablet_id = t->tablet_id(); *max_score = s; diff --git a/be/src/cloud/cloud_tablets_channel.cpp b/be/src/cloud/cloud_tablets_channel.cpp index 775222133f4419..9a5f25c0439aeb 100644 --- a/be/src/cloud/cloud_tablets_channel.cpp +++ b/be/src/cloud/cloud_tablets_channel.cpp @@ -38,14 +38,44 @@ CloudTabletsChannel::~CloudTabletsChannel() = default; std::unique_ptr CloudTabletsChannel::create_delta_writer( const WriteRequest& request) { - return std::make_unique(_engine, request, _profile, _load_id); + DCHECK(request.write_req_type == WriteRequestType::DATA); + DCHECK(request.table_schema_param != nullptr); + + if (request.binlog_tablet_id <= 0) { + return std::make_unique(_engine, request, _profile, _load_id); + } + + int64_t row_binlog_index_id = 0; + for (const auto* index_schema : request.table_schema_param->indexes()) { + if (index_schema->index_id == request.index_id) { + row_binlog_index_id = index_schema->row_binlog_id; + break; + } + } + DCHECK(row_binlog_index_id > 0); + + const auto* row_binlog_index_schema = + request.table_schema_param->row_binlog_index_schema(row_binlog_index_id); + DCHECK(row_binlog_index_schema != nullptr); + + WriteRequest group_build_req = request; + group_build_req.write_req_type = WriteRequestType::GROUP; + + WriteRequest sub_data_req = request; + sub_data_req.write_req_type = WriteRequestType::DATA; + + WriteRequest sub_row_binlog_req = request; + sub_row_binlog_req.write_req_type = WriteRequestType::ROW_BINLOG; + sub_row_binlog_req.tablet_id = request.binlog_tablet_id; + sub_row_binlog_req.index_id = row_binlog_index_schema->index_id; + sub_row_binlog_req.schema_hash = row_binlog_index_schema->schema_hash; + + return std::make_unique(_engine, group_build_req, sub_data_req, + sub_row_binlog_req, _profile, _load_id); } Status CloudTabletsChannel::add_batch(const PTabletWriterAddBlockRequest& request, PTabletWriterAddBlockResult* response) { - if (_schema != nullptr && _schema->row_binlog_index_schema() != nullptr) { - return Status::NotSupported("cloud mode does not support binlog now"); - } // FIXME(plat1ko): Too many duplicate code with `TabletsChannel` SCOPED_TIMER(_add_batch_timer); int64_t cur_seq = 0; @@ -293,7 +323,7 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo it++; } - tablet_vec->Reserve(static_cast(writers_to_commit.size())); + tablet_vec->Reserve(static_cast(writers_to_commit.size() * 2)); for (auto* writer : writers_to_commit) { PTabletInfo* tablet_info = tablet_vec->Add(); tablet_info->set_tablet_id(writer->tablet_id()); @@ -301,6 +331,13 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo tablet_info->set_schema_hash(0); tablet_info->set_received_rows(writer->total_received_rows()); tablet_info->set_num_rows_filtered(writer->num_rows_filtered()); + if (writer->binlog_tablet_id() > 0) { + PTabletInfo* binlog_tablet_info = tablet_vec->Add(); + binlog_tablet_info->set_tablet_id(writer->binlog_tablet_id()); + binlog_tablet_info->set_schema_hash(0); + binlog_tablet_info->set_received_rows(writer->total_received_rows()); + binlog_tablet_info->set_num_rows_filtered(writer->num_rows_filtered()); + } // These stats may be larger than the actual value if the txn is aborted writer->update_tablet_stats(); } diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp index b6349067b87903..d1fb847a021c9b 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp @@ -59,7 +59,8 @@ Status CloudTxnDeleteBitmapCache::get_tablet_txn_info( TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset, DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration, std::shared_ptr* partial_update_info, - std::shared_ptr* publish_status, TxnPublishInfo* previous_publish_info) { + std::shared_ptr* publish_status, TxnPublishInfo* previous_publish_info, + RowBinlogTxnInfo* attach_row_binlog) { { std::shared_lock rlock(_rwlock); TxnKey key(transaction_id, tablet_id); @@ -79,6 +80,9 @@ Status CloudTxnDeleteBitmapCache::get_tablet_txn_info( *partial_update_info = iter->second.partial_update_info; *publish_status = iter->second.publish_status; *previous_publish_info = iter->second.publish_info; + if (attach_row_binlog != nullptr) { + *attach_row_binlog = iter->second.attach_row_binlog; + } } auto st = get_delete_bitmap(transaction_id, tablet_id, delete_bitmap, rowset_ids, nullptr); @@ -186,7 +190,8 @@ Status CloudTxnDeleteBitmapCache::get_delete_bitmap( void CloudTxnDeleteBitmapCache::set_tablet_txn_info( TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids, RowsetSharedPtr rowset, int64_t txn_expiration, - std::shared_ptr partial_update_info) { + std::shared_ptr partial_update_info, + const RowBinlogTxnInfo& attach_row_binlog) { int64_t txn_expiration_min = duration_cast(std::chrono::system_clock::now().time_since_epoch()) .count() + @@ -198,7 +203,7 @@ void CloudTxnDeleteBitmapCache::set_tablet_txn_info( std::shared_ptr publish_status = std::make_shared(PublishStatus::INIT); _txn_map[txn_key] = TxnVal(rowset, txn_expiration, std::move(partial_update_info), - std::move(publish_status)); + std::move(publish_status), attach_row_binlog); _expiration_txn.emplace(txn_expiration, txn_key); } std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id); @@ -359,4 +364,4 @@ void CloudTxnDeleteBitmapCache::_clean_thread_callback() { std::chrono::seconds(config::remove_expired_tablet_txn_info_interval_seconds))); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.h b/be/src/cloud/cloud_txn_delete_bitmap_cache.h index 7cf6c27ecd8491..6be96990ed7f2b 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.h +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.h @@ -43,12 +43,14 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration, std::shared_ptr* partial_update_info, std::shared_ptr* publish_status, - TxnPublishInfo* previous_publish_info); + TxnPublishInfo* previous_publish_info, + RowBinlogTxnInfo* attach_row_binlog = nullptr); void set_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids, RowsetSharedPtr rowset, int64_t txn_expirationm, - std::shared_ptr partial_update_info); + std::shared_ptr partial_update_info, + const RowBinlogTxnInfo& attach_row_binlog = {}); Status update_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, @@ -109,14 +111,17 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { std::shared_ptr publish_status = nullptr; // used to determine if the retry needs to re-calculate the delete bitmap TxnPublishInfo publish_info; + RowBinlogTxnInfo attach_row_binlog; TxnVal() : txn_expiration(0) {}; TxnVal(RowsetSharedPtr rowset_, int64_t txn_expiration_, std::shared_ptr partial_update_info_, - std::shared_ptr publish_status_) + std::shared_ptr publish_status_, + const RowBinlogTxnInfo& attach_row_binlog_) : rowset(std::move(rowset_)), txn_expiration(txn_expiration_), partial_update_info(std::move(partial_update_info_)), - publish_status(std::move(publish_status_)) {} + publish_status(std::move(publish_status_)), + attach_row_binlog(attach_row_binlog_) {} }; std::map _txn_map; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index f03f28c0c85a37..8beeb1ba806536 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -49,6 +49,7 @@ DEFINE_mInt32(lease_compaction_interval_seconds, "20"); DEFINE_mBool(enable_parallel_cumu_compaction, "false"); DEFINE_mDouble(base_compaction_thread_num_factor, "0.25"); DEFINE_mDouble(cumu_compaction_thread_num_factor, "0.5"); +DEFINE_mDouble(binlog_compaction_thread_num_factor, "0.25"); DEFINE_mInt32(check_auto_compaction_interval_seconds, "5"); DEFINE_mInt32(max_base_compaction_task_num_per_disk, "2"); DEFINE_mBool(prioritize_query_perf_in_compaction, "false"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 63355451c8c218..15162b04dc05ca 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -90,6 +90,7 @@ DECLARE_mInt32(lease_compaction_interval_seconds); DECLARE_mBool(enable_parallel_cumu_compaction); DECLARE_mDouble(base_compaction_thread_num_factor); DECLARE_mDouble(cumu_compaction_thread_num_factor); +DECLARE_mDouble(binlog_compaction_thread_num_factor); DECLARE_mInt32(check_auto_compaction_interval_seconds); DECLARE_mInt32(max_base_compaction_task_num_per_disk); DECLARE_mBool(prioritize_query_perf_in_compaction); diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 5c67e8e0f82f2b..151aa9eaa7842d 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -762,18 +762,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in) if (in.has_binlog_config()) { out->mutable_binlog_config()->CopyFrom(in.binlog_config()); } - if (in.has_row_binlog_schema()) { - doris_tablet_schema_to_cloud(out->mutable_row_binlog_schema(), in.row_binlog_schema()); - } - if (in.row_binlog_rs_metas_size()) { - out->mutable_row_binlog_rs_metas()->Reserve(in.row_binlog_rs_metas_size()); - for (const auto& rs_meta : in.row_binlog_rs_metas()) { - doris_rowset_meta_to_cloud(out->add_row_binlog_rs_metas(), rs_meta); - } - } - if (in.has_row_binlog_schema_hash()) { - out->set_row_binlog_schema_hash(in.row_binlog_schema_hash()); - } + out->set_is_row_binlog_tablet(in.is_row_binlog_tablet()); out->set_compaction_policy(in.compaction_policy()); out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes()); out->set_time_series_compaction_file_count_threshold( @@ -853,21 +842,7 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) { if (in.has_binlog_config()) { out->mutable_binlog_config()->Swap(in.mutable_binlog_config()); } - if (in.has_row_binlog_schema()) { - doris_tablet_schema_to_cloud(out->mutable_row_binlog_schema(), - std::move(*in.mutable_row_binlog_schema())); - } - if (in.row_binlog_rs_metas_size()) { - int row_binlog_rs_metas_size = in.row_binlog_rs_metas_size(); - out->mutable_row_binlog_rs_metas()->Reserve(row_binlog_rs_metas_size); - for (int i = 0; i < row_binlog_rs_metas_size; ++i) { - doris_rowset_meta_to_cloud(out->add_row_binlog_rs_metas(), - std::move(*in.mutable_row_binlog_rs_metas(i))); - } - } - if (in.has_row_binlog_schema_hash()) { - out->set_row_binlog_schema_hash(in.row_binlog_schema_hash()); - } + out->set_is_row_binlog_tablet(in.is_row_binlog_tablet()); out->set_compaction_policy(in.compaction_policy()); out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes()); out->set_time_series_compaction_file_count_threshold( @@ -954,18 +929,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in) if (in.has_binlog_config()) { out->mutable_binlog_config()->CopyFrom(in.binlog_config()); } - if (in.has_row_binlog_schema()) { - cloud_tablet_schema_to_doris(out->mutable_row_binlog_schema(), in.row_binlog_schema()); - } - if (in.row_binlog_rs_metas_size()) { - out->mutable_row_binlog_rs_metas()->Reserve(in.row_binlog_rs_metas_size()); - for (const auto& rs_meta : in.row_binlog_rs_metas()) { - cloud_rowset_meta_to_doris(out->add_row_binlog_rs_metas(), rs_meta); - } - } - if (in.has_row_binlog_schema_hash()) { - out->set_row_binlog_schema_hash(in.row_binlog_schema_hash()); - } + out->set_is_row_binlog_tablet(in.is_row_binlog_tablet()); out->set_compaction_policy(in.compaction_policy()); out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes()); out->set_time_series_compaction_file_count_threshold( @@ -1045,21 +1009,7 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) { if (in.has_binlog_config()) { out->mutable_binlog_config()->Swap(in.mutable_binlog_config()); } - if (in.has_row_binlog_schema()) { - cloud_tablet_schema_to_doris(out->mutable_row_binlog_schema(), - std::move(*in.mutable_row_binlog_schema())); - } - if (in.row_binlog_rs_metas_size()) { - int row_binlog_rs_metas_size = in.row_binlog_rs_metas_size(); - out->mutable_row_binlog_rs_metas()->Reserve(row_binlog_rs_metas_size); - for (int i = 0; i < row_binlog_rs_metas_size; ++i) { - cloud_rowset_meta_to_doris(out->add_row_binlog_rs_metas(), - std::move(*in.mutable_row_binlog_rs_metas(i))); - } - } - if (in.has_row_binlog_schema_hash()) { - out->set_row_binlog_schema_hash(in.row_binlog_schema_hash()); - } + out->set_is_row_binlog_tablet(in.is_row_binlog_tablet()); out->set_compaction_policy(in.compaction_policy()); out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes()); out->set_time_series_compaction_file_count_threshold( diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9f697a2dcbed81..db06b6325786bf 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -484,9 +484,6 @@ DEFINE_mInt32(binlog_compaction_task_num_per_disk, "4"); DEFINE_mInt32(binlog_compaction_file_count_threshold, "100"); DEFINE_mInt32(binlog_level_compaction_max_deltas, "2000"); DEFINE_mInt64(binlog_compaction_time_threshold_seconds, "3600"); -DEFINE_mInt32(binlog_compaction_permits_percent, "30"); -DEFINE_Validator(binlog_compaction_permits_percent, - [](const int config) -> bool { return config >= 1 && config <= 80; }); DEFINE_mInt32(max_binlog_compaction_threads, "-1"); DEFINE_Bool(enable_base_compaction_idle_sched, "true"); diff --git a/be/src/common/config.h b/be/src/common/config.h index f5c72eb776d5d3..92f556984a970d 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -554,7 +554,6 @@ DECLARE_mInt32(binlog_compaction_task_num_per_disk); DECLARE_mInt32(binlog_compaction_file_count_threshold); DECLARE_mInt32(binlog_level_compaction_max_deltas); DECLARE_mInt64(binlog_compaction_time_threshold_seconds); -DECLARE_mInt32(binlog_compaction_permits_percent); DECLARE_mInt32(max_binlog_compaction_threads); DECLARE_Bool(enable_base_compaction_idle_sched); diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index de596407e512b3..36bebcddbb30ad 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -202,7 +202,6 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_rowsets_num, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_segments_num, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_used_permits, MetricUnit::NOUNIT); -DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(binlog_compaction_used_permits, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_waitting_permits, MetricUnit::NOUNIT); DEFINE_HISTOGRAM_METRIC_PROTOTYPE_2ARG(tablet_version_num_distribution, MetricUnit::NOUNIT); @@ -400,7 +399,6 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_segments_num); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_used_permits); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, binlog_compaction_used_permits); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_waitting_permits); HISTOGRAM_METRIC_REGISTER(_server_metric_entity, tablet_version_num_distribution); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 1679f28d775a8d..aadda83fa86d3d 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -176,7 +176,6 @@ class DorisMetrics { // permits have been used for all compaction tasks IntGauge* compaction_used_permits = nullptr; - IntGauge* binlog_compaction_used_permits = nullptr; // permits required by the compaction task which is waiting for permits IntGauge* compaction_waitting_permits = nullptr; diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 50d41d3dcc979a..ef5172df022b53 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -1019,8 +1019,6 @@ Status OlapScanLocalState::prepare(RuntimeState* state) { {0, _tablets[i].version}, {.skip_missing_versions = _state->skip_missing_version(), .enable_fetch_rowsets_from_peers = config::enable_fetch_rowsets_from_peer_replicas, - .capture_row_binlog = olap_scan_node().__isset.read_row_binlog && - olap_scan_node().read_row_binlog, .enable_prefer_cached_rowset = config::is_cloud_mode() ? _state->enable_prefer_cached_rowset() : false, .query_freshness_tolerance_ms = diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 834ee012737ecd..90754036fbc8d2 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -78,8 +78,8 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param _key_ranges(std::move(params.key_ranges)), _tablet_reader_params({.tablet = std::move(params.tablet), .tablet_schema {}, - .reader_type = params.read_row_binlog ? ReaderType::READER_BINLOG - : ReaderType::READER_QUERY, + .reader_type = ReaderType::READER_QUERY, + .read_row_binlog = params.read_row_binlog, .aggregation = params.aggregation, .version = {0, params.version}, .start_key {}, @@ -213,10 +213,7 @@ Status OlapScanner::_prepare_impl() { _tablet_reader->set_preferred_block_size_bytes(_state->preferred_block_size_bytes()); { TOlapScanNode& olap_scan_node = local_state->olap_scan_node(); - TabletSchemaSPtr source_tablet_schema = - _tablet_reader_params.reader_type == ReaderType::READER_BINLOG - ? tablet->row_binlog_tablet_schema() - : tablet->tablet_schema(); + TabletSchemaSPtr source_tablet_schema = tablet->tablet_schema(); tablet_schema = std::make_shared(); tablet_schema->copy_from(*source_tablet_schema); @@ -252,8 +249,6 @@ Status OlapScanner::_prepare_impl() { .skip_missing_versions = _state->skip_missing_version(), .enable_fetch_rowsets_from_peers = config::enable_fetch_rowsets_from_peer_replicas, - .capture_row_binlog = - _tablet_reader_params.reader_type == ReaderType::READER_BINLOG, .enable_prefer_cached_rowset = config::is_cloud_mode() ? _state->enable_prefer_cached_rowset() : false, @@ -337,12 +332,10 @@ Status OlapScanner::_init_tso_predicates() { } auto& tablet_schema = _tablet_reader_params.tablet_schema; - int32_t tso_index = _tablet_reader_params.reader_type == ReaderType::READER_BINLOG - ? tablet_schema->binlog_tso_col_idx() - : tablet_schema->commit_tso_col_idx(); - const std::string& column_name = _tablet_reader_params.reader_type == ReaderType::READER_BINLOG - ? BINLOG_TSO_COL - : COMMIT_TSO_COL; + int32_t tso_index = _tablet_reader_params.read_row_binlog ? tablet_schema->binlog_tso_col_idx() + : tablet_schema->commit_tso_col_idx(); + const std::string& column_name = + _tablet_reader_params.read_row_binlog ? BINLOG_TSO_COL : COMMIT_TSO_COL; if (tso_index < 0) { return Status::InternalError("Column {} not found in tablet schema after append", column_name); diff --git a/be/src/exec/sink/writer/vtablet_writer.cpp b/be/src/exec/sink/writer/vtablet_writer.cpp index 5bd43f88726625..b41363fa1ae69a 100644 --- a/be/src/exec/sink/writer/vtablet_writer.cpp +++ b/be/src/exec/sink/writer/vtablet_writer.cpp @@ -772,6 +772,12 @@ void VNodeChannel::_open_internal(bool is_incremental) { auto* ptablet = request->add_tablets(); ptablet->set_partition_id(tablet.partition_id); ptablet->set_tablet_id(tablet.tablet_id); + // only write binlog on backends that also own the binlog tablet. + int64_t binlog_tablet_id = + _parent->_location->get_binlog_tablet_id(tablet.tablet_id, _node_id); + if (binlog_tablet_id > 0) { + ptablet->set_binlog_tablet_id(binlog_tablet_id); + } deduper.insert(tablet.tablet_id); _all_tablets.push_back(std::move(tablet)); } diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index f42e486eb40aea..c085f93347850f 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -33,9 +33,7 @@ enum class ReaderType : uint8_t { READER_COLD_DATA_COMPACTION = 5, READER_SEGMENT_COMPACTION = 6, READER_FULL_COMPACTION = 7, - READER_BINLOG_COMPACTION = 8, - READER_BINLOG = 9, - UNKNOWN = 10 + UNKNOWN = 8 }; namespace io { diff --git a/be/src/load/channel/tablets_channel.cpp b/be/src/load/channel/tablets_channel.cpp index 1624a5e08c140e..cd1ca1e6097084 100644 --- a/be/src/load/channel/tablets_channel.cpp +++ b/be/src/load/channel/tablets_channel.cpp @@ -248,6 +248,9 @@ Status BaseTabletsChannel::incremental_open(const PTabletWriterOpenRequest& para wrequest.write_file_cache = params.write_file_cache(); wrequest.storage_vault_id = params.storage_vault_id(); wrequest.enable_table_memtable_backpressure = params.is_adaptive_random_bucket(); + if (tablet.has_binlog_tablet_id()) { + wrequest.binlog_tablet_id = tablet.binlog_tablet_id(); + } auto delta_writer = create_delta_writer(wrequest); { @@ -302,6 +305,12 @@ std::unique_ptr TabletsChannel::create_delta_writer(const Write DCHECK(request.write_req_type == WriteRequestType::DATA); DCHECK(request.table_schema_param != nullptr); + // whether to write binlog is decided by binlog_tablet_id: it is set only when this backend + // also owns the binlog tablet that is paired with the base tablet. + if (request.binlog_tablet_id <= 0) { + return std::make_unique(_engine, request, _profile, _load_id); + } + int64_t row_binlog_index_id = 0; for (const auto* index_schema : request.table_schema_param->indexes()) { if (index_schema->index_id == request.index_id) { @@ -309,13 +318,11 @@ std::unique_ptr TabletsChannel::create_delta_writer(const Write break; } } - if (row_binlog_index_id <= 0) { - return std::make_unique(_engine, request, _profile, _load_id); - } + DCHECK(row_binlog_index_id > 0); - const auto* row_binlog_index_schema = request.table_schema_param->row_binlog_index_schema(); + const auto* row_binlog_index_schema = + request.table_schema_param->row_binlog_index_schema(row_binlog_index_id); DCHECK(row_binlog_index_schema != nullptr); - DCHECK(row_binlog_index_schema->index_id == row_binlog_index_id); // group_build_req is only for the group wrapper itself. It provides the group semantics and // metadata used by BaseDeltaWriter/GroupRowsetBuilder to expose tablet_id, txn_id, @@ -329,6 +336,7 @@ std::unique_ptr TabletsChannel::create_delta_writer(const Write WriteRequest sub_row_binlog_req = request; sub_row_binlog_req.write_req_type = WriteRequestType::ROW_BINLOG; + sub_row_binlog_req.tablet_id = request.binlog_tablet_id; sub_row_binlog_req.index_id = row_binlog_index_schema->index_id; sub_row_binlog_req.schema_hash = row_binlog_index_schema->schema_hash; @@ -502,6 +510,14 @@ void TabletsChannel::_commit_txn(DeltaWriter* writer, const PTabletWriterAddBloc tablet_info->set_schema_hash(0); tablet_info->set_received_rows(writer->total_received_rows()); tablet_info->set_num_rows_filtered(writer->num_rows_filtered()); + // report the row binlog tablet as a normal tablet so FE advances its version. + if (writer->binlog_tablet_id() > 0) { + PTabletInfo* binlog_tablet_info = tablet_vec->Add(); + binlog_tablet_info->set_tablet_id(writer->binlog_tablet_id()); + binlog_tablet_info->set_schema_hash(0); + binlog_tablet_info->set_received_rows(writer->total_received_rows()); + binlog_tablet_info->set_num_rows_filtered(writer->num_rows_filtered()); + } _total_received_rows += writer->total_received_rows(); _num_rows_filtered += writer->num_rows_filtered(); } else { @@ -600,6 +616,9 @@ Status BaseTabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& req .storage_vault_id = request.storage_vault_id(), .enable_table_memtable_backpressure = request.is_adaptive_random_bucket(), }; + if (tablet.has_binlog_tablet_id()) { + wrequest.binlog_tablet_id = tablet.binlog_tablet_id(); + } auto delta_writer = create_delta_writer(wrequest); { diff --git a/be/src/load/delta_writer/delta_writer.h b/be/src/load/delta_writer/delta_writer.h index 9995a5e5480ff7..b2e6f2893e66e5 100644 --- a/be/src/load/delta_writer/delta_writer.h +++ b/be/src/load/delta_writer/delta_writer.h @@ -90,6 +90,8 @@ class BaseDeltaWriter { int64_t tablet_id() const { return _req.tablet_id; } + int64_t binlog_tablet_id() const { return _req.binlog_tablet_id; } + int64_t txn_id() const { return _req.txn_id; } int64_t total_received_rows() const { return _memtable_writer->total_received_rows(); } diff --git a/be/src/load/delta_writer/delta_writer_context.h b/be/src/load/delta_writer/delta_writer_context.h index 5b7e3f144b80ea..856601c0be6a83 100644 --- a/be/src/load/delta_writer/delta_writer_context.h +++ b/be/src/load/delta_writer/delta_writer_context.h @@ -44,6 +44,8 @@ struct WriteRequest { int64_t txn_expiration = 0; // For cloud mode int64_t index_id = 0; int64_t partition_id = 0; + // the row binlog tablet id written together with this base tablet, 0 means no binlog. + int64_t binlog_tablet_id = 0; PUniqueId load_id; TupleDescriptor* tuple_desc = nullptr; // slots are in order of tablet's schema diff --git a/be/src/load/stream_load/stream_load_context.cpp b/be/src/load/stream_load/stream_load_context.cpp index 119281b00fffd4..f75812cf26dcda 100644 --- a/be/src/load/stream_load/stream_load_context.cpp +++ b/be/src/load/stream_load/stream_load_context.cpp @@ -377,4 +377,9 @@ bool StreamLoadContext::is_mow_table() const { put_result.pipeline_params.is_mow_table; } +bool StreamLoadContext::enable_tso() const { + return put_result.__isset.pipeline_params && put_result.pipeline_params.__isset.enable_tso && + put_result.pipeline_params.enable_tso; +} + } // namespace doris diff --git a/be/src/load/stream_load/stream_load_context.h b/be/src/load/stream_load/stream_load_context.h index 2eab22dc7d4513..963a7c740881b0 100644 --- a/be/src/load/stream_load/stream_load_context.h +++ b/be/src/load/stream_load/stream_load_context.h @@ -159,6 +159,7 @@ class StreamLoadContext { std::string brief(bool detail = false) const; bool is_mow_table() const; + bool enable_tso() const; Status allocate_schema_buffer() { if (_schema_buffer == nullptr) { diff --git a/be/src/service/http/action/compaction_action.cpp b/be/src/service/http/action/compaction_action.cpp index 83371b0995fd28..80638ca5ef121a 100644 --- a/be/src/service/http/action/compaction_action.cpp +++ b/be/src/service/http/action/compaction_action.cpp @@ -37,7 +37,6 @@ #include "service/http/http_request.h" #include "service/http/http_status.h" #include "storage/compaction/base_compaction.h" -#include "storage/compaction/binlog_compaction.h" #include "storage/compaction/cumulative_compaction.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" @@ -136,7 +135,7 @@ Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j std::string compaction_type = req->param(PARAM_COMPACTION_TYPE); if (compaction_type != PARAM_COMPACTION_BASE && compaction_type != PARAM_COMPACTION_CUMULATIVE && - compaction_type != PARAM_COMPACTION_FULL && compaction_type != PARAM_COMPACTION_BINLOG) { + compaction_type != PARAM_COMPACTION_FULL) { return Status::NotSupported("The compaction type '{}' is not supported", compaction_type); } @@ -180,7 +179,7 @@ Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j RETURN_IF_ERROR(_engine.submit_compaction_task(tablet, CompactionType::FULL_COMPACTION, force, true, 1)); } else { - // 3. execute base/cumulative/binlog compaction task in a detached thread + // 3. execute base/cumulative compaction task in a detached thread std::packaged_task task([this, tablet, compaction_type]() { return _execute_compaction_callback(tablet, compaction_type); }); @@ -271,20 +270,6 @@ Status CompactionAction::_handle_run_status_compaction(HttpRequest* req, std::st } } - { - // use try lock to check this tablet is running binlog compaction - std::unique_lock lock_binlog(tablet->get_binlog_compaction_lock(), - std::try_to_lock); - if (!lock_binlog.owns_lock()) { - msg = "compaction task for this tablet is running"; - compaction_type = "binlog"; - run_status = true; - *json_result = absl::Substitute(json_template, run_status, msg, tablet_id, - compaction_type); - return Status::OK(); - } - } - { // use try lock to check this tablet is running base compaction std::unique_lock lock_base(tablet->get_base_compaction_lock(), @@ -305,15 +290,16 @@ Status CompactionAction::_handle_run_status_compaction(HttpRequest* req, std::st } Status CompactionAction::_execute_compaction_callback(TabletSharedPtr tablet, - const std::string& compaction_type, - int8_t prefer_compaction_level) { + const std::string& compaction_type) { MonotonicStopWatch timer; timer.start(); std::shared_ptr cumulative_compaction_policy = CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( tablet->tablet_meta()->compaction_policy()); - if (tablet->get_cumulative_compaction_policy() == nullptr) { + if (tablet->get_cumulative_compaction_policy() == nullptr || + tablet->get_cumulative_compaction_policy()->name() != + cumulative_compaction_policy->name()) { tablet->set_cumulative_compaction_policy(cumulative_compaction_policy); } Status res = Status::OK(); @@ -380,29 +366,6 @@ Status CompactionAction::_execute_compaction_callback(TabletSharedPtr tablet, } } } - } else if (compaction_type == PARAM_COMPACTION_BINLOG) { - if (prefer_compaction_level < 0) { - tablet->calc_compaction_score(CompactionType::BINLOG_COMPACTION, - &prefer_compaction_level); - } - if (prefer_compaction_level < 0) { - res = Status::Error( - "failed to init binlog compaction due to no suitable version"); - } else { - BinlogCompaction binlog_compaction(_engine, tablet, prefer_compaction_level); - res = do_compact(binlog_compaction, CompactionProfileType::BINLOG); - if (!res) { - if (res.is()) { - VLOG_NOTICE << "failed to init binlog compaction due to no suitable version," - << "tablet=" << tablet->tablet_id() << ", compaction_level=" - << static_cast(prefer_compaction_level); - } else { - LOG(WARNING) << "failed to do binlog compaction. res=" << res - << ", table=" << tablet->tablet_id() << ", compaction_level=" - << static_cast(prefer_compaction_level); - } - } - } } timer.stop(); LOG(INFO) << "Manual compaction task finish, status=" << res diff --git a/be/src/service/http/action/compaction_action.h b/be/src/service/http/action/compaction_action.h index d22083e6e99a6b..0d06627443abd7 100644 --- a/be/src/service/http/action/compaction_action.h +++ b/be/src/service/http/action/compaction_action.h @@ -40,7 +40,6 @@ const std::string PARAM_COMPACTION_TYPE = "compact_type"; const std::string PARAM_COMPACTION_BASE = "base"; const std::string PARAM_COMPACTION_CUMULATIVE = "cumulative"; const std::string PARAM_COMPACTION_FULL = "full"; -const std::string PARAM_COMPACTION_BINLOG = "binlog"; const std::string PARAM_COMPACTION_FORCE = "force"; /// This action is used for viewing the compaction status. @@ -58,12 +57,11 @@ class CompactionAction : public HttpHandlerWithAuth { Status _handle_show_compaction(HttpRequest* req, std::string* json_result); /// execute compaction request to run compaction task - /// param compact_type in req to distinguish the task type, base/cumulative/full/binlog + /// param compact_type in req to distinguish the task type, base/cumulative/full Status _handle_run_compaction(HttpRequest* req, std::string* json_result); /// thread callback function for the tablet to do base/cumulative/binlog compaction - Status _execute_compaction_callback(TabletSharedPtr tablet, const std::string& compaction_type, - int8_t prefer_compaction_level = -1); + Status _execute_compaction_callback(TabletSharedPtr tablet, const std::string& compaction_type); /// fetch compaction running status Status _handle_run_status_compaction(HttpRequest* req, std::string* json_result); diff --git a/be/src/service/http/action/compaction_profile_action.cpp b/be/src/service/http/action/compaction_profile_action.cpp index a91744adad3b1b..066ba4d0456cae 100644 --- a/be/src/service/http/action/compaction_profile_action.cpp +++ b/be/src/service/http/action/compaction_profile_action.cpp @@ -106,10 +106,10 @@ void CompactionProfileAction::handle(HttpRequest* req) { // compact_type compact_type = req->param("compact_type"); if (!compact_type.empty() && compact_type != "base" && compact_type != "cumulative" && - compact_type != "full" && compact_type != "binlog") { + compact_type != "full") { HttpChannel::send_reply( req, HttpStatus::BAD_REQUEST, - R"({"status":"Error","msg":"compact_type must be one of: base, cumulative, full, binlog"})"); + R"({"status":"Error","msg":"compact_type must be one of: base, cumulative, full"})"); return; } diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 67df019be260cf..ad380d96450374 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -2577,7 +2577,7 @@ void PInternalService::get_tablet_rowsets(google::protobuf::RpcController* contr response->mutable_rowsets()->Add(std::move(meta)); } if (request->has_delete_bitmap_keys()) { - DCHECK(tablet->enable_unique_key_merge_on_write()); + DCHECK(tablet->need_read_delete_bitmap()); auto delete_bitmap = std::move(ret.value().delete_bitmap); auto keys_pb = request->delete_bitmap_keys(); size_t len = keys_pb.rowset_ids().size(); diff --git a/be/src/storage/binlog.h b/be/src/storage/binlog.h index 330fb8a9bf8989..0907c55a97042d 100644 --- a/be/src/storage/binlog.h +++ b/be/src/storage/binlog.h @@ -33,6 +33,7 @@ #include "exec/sink/autoinc_buffer.h" #include "storage/olap_common.h" #include "storage/olap_define.h" // DataWriteType +#include "storage/tablet/tablet_fwd.h" // BaseTabletSPtr #include "storage/tablet/tablet_schema.h" // TabletSchemaSPtr #include "storage/utils.h" // BINLOG_*_COL @@ -124,16 +125,6 @@ inline std::string get_binlog_data_key_from_meta_key(const std::string_view meta return fmt::format("{}data_{}", kBinlogPrefix, meta_key.substr(kBinlogMetaPrefix.length())); } -inline auto make_row_binlog_key_prefix(const TabletUid& tablet_uid, const RowsetId& rowset_id) { - return fmt::format("{}{}_{}_", kRowBinlogPrefix, tablet_uid.to_string(), rowset_id.to_string()); -} - -inline auto make_row_binlog_key(const TabletUid& tablet_uid, const RowsetId& rowset_id, - const RowsetId& binlog_rowset_id) { - return fmt::format("{}{}_{}_{}", kRowBinlogPrefix, tablet_uid.to_string(), - rowset_id.to_string(), binlog_rowset_id.to_string()); -} - // Allocate per-row LSNs for row-binlog data. // The caller must provide a valid auto-inc buffer (typically from GlobalAutoIncBuffers). inline Status allocate_binlog_lsn(const std::shared_ptr& lsn_buffer, @@ -208,6 +199,7 @@ struct SegmentWriteBinlogOptions { // source context, used for retrieving historical row and building binlog block struct SourceWriteDataOptions { + BaseTabletSPtr base_tablet = nullptr; TabletSchemaSPtr tablet_schema = nullptr; std::shared_ptr partial_update_info; std::shared_ptr mow_context; diff --git a/be/src/storage/compaction/binlog_compaction.cpp b/be/src/storage/compaction/binlog_compaction.cpp deleted file mode 100644 index 7ee72cf7615907..00000000000000 --- a/be/src/storage/compaction/binlog_compaction.cpp +++ /dev/null @@ -1,163 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/binlog_compaction.h" - -#include - -#include -#include -#include - -#include "common/logging.h" -#include "runtime/thread_context.h" -#include "storage/compaction/binlog_compaction_policy.h" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_meta.h" -#include "storage/tablet/tablet.h" -#include "util/defer_op.h" -#include "util/time.h" -#include "util/trace.h" - -namespace doris { -using namespace ErrorCode; - -BinlogCompaction::BinlogCompaction(StorageEngine& engine, const TabletSharedPtr& tablet, - int8_t compaction_level) - : CompactionMixin(engine, tablet, - "BinlogCompaction:" + std::to_string(tablet->tablet_id())), - _compaction_level(compaction_level) {} - -BinlogCompaction::~BinlogCompaction() = default; - -Status BinlogCompaction::prepare_compact() { - Status st; - Defer defer_set_st([&] { - if (!st.ok()) { - tablet()->set_last_binlog_compaction_status(st.to_string()); - } - }); - - if (!tablet()->init_succeeded()) { - st = Status::Error("_tablet init failed"); - return st; - } - - std::unique_lock lock(tablet()->get_binlog_compaction_lock(), std::try_to_lock); - if (!lock.owns_lock()) { - st = Status::Error( - "The tablet is under binlog compaction. tablet={}", _tablet->tablet_id()); - return st; - } - - st = pick_rowsets_to_compact(); - RETURN_IF_ERROR(st); - - COUNTER_UPDATE(_input_rowsets_counter, _input_rowsets.size()); - - st = Status::OK(); - return st; -} - -Status BinlogCompaction::execute_compact() { - Status st; - Defer defer_set_st([&] { - tablet()->set_last_binlog_compaction_status(st.to_string()); - if (!st.ok()) { - tablet()->set_last_binlog_compaction_failure_time(UnixMillis()); - } else { - tablet()->set_last_binlog_compaction_success_time(_compaction_level, UnixMillis()); - } - }); - - std::unique_lock lock(tablet()->get_binlog_compaction_lock(), std::try_to_lock); - if (!lock.owns_lock()) { - st = Status::Error( - "The tablet is under binlog compaction. tablet={}", _tablet->tablet_id()); - return st; - } - - SCOPED_ATTACH_TASK(_mem_tracker); - - st = CompactionMixin::execute_compact(); - RETURN_IF_ERROR(st); - - TEST_SYNC_POINT_RETURN_WITH_VALUE("binlog_compaction::BinlogCompaction::execute_compact", - Status::OK()); - - DCHECK_EQ(_state, CompactionState::SUCCESS); - - st = Status::OK(); - return st; -} - -Status BinlogCompaction::pick_rowsets_to_compact() { - auto candidate_rowsets = tablet()->pick_candidate_rowsets_to_binlog_compaction(); - if (candidate_rowsets.empty()) { - return Status::Error("candidate_rowsets is empty"); - } - - // candidate_rowsets may not be continuous - // So we need to choose the longest continuous path from it. - std::vector missing_versions; - find_longest_consecutive_version(&candidate_rowsets, &missing_versions); - if (!missing_versions.empty()) { - DCHECK(missing_versions.size() % 2 == 0); - LOG(WARNING) << "There are missed versions among binlog rowsets. " - << "total missed version size: " << missing_versions.size() / 2 - << ", first missed version prev rowset verison=" << missing_versions[0] - << ", first missed version next rowset version=" << missing_versions[1] - << ", tablet=" << _tablet->tablet_id(); - } - - tablet()->binlog_compaction_policy()->pick_input_rowsets(tablet(), candidate_rowsets, - _compaction_level, &_input_rowsets); - - // Binlog compaction will process with at least 1 rowset. - // So when there is no rowset being chosen, we should return Status::Error(): - if (_input_rowsets.empty()) { - VLOG_DEBUG << "binlog compaction can't get input rowsets, tablet=" << _tablet->tablet_id(); - return Status::Error("_input_rowsets is empty"); - } - - return Status::OK(); -} - -Status BinlogCompaction::modify_rowsets() { - std::vector output_rowsets; - output_rowsets.push_back(_output_rowset); - - if (_is_ordered_data_compaction && - _compaction_level == BinlogCompactionPolicy::kBinlogCompactionMaxLevel - 1) { - _output_rowset->rowset_meta()->set_segments_overlap(OVERLAPPING); - } - tablet()->binlog_compaction_policy()->update_compaction_level(tablet(), _input_rowsets, - _output_rowset); - { - std::lock_guard wrlock_(tablet()->get_rowset_update_lock()); - std::lock_guard wrlock(_tablet->get_header_lock()); - SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD); - RETURN_IF_ERROR(tablet()->modify_row_binlog_rowsets(output_rowsets, _input_rowsets)); - } - { - std::shared_lock rlock(_tablet->get_header_lock()); - tablet()->save_meta(); - } - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/storage/compaction/binlog_compaction.h b/be/src/storage/compaction/binlog_compaction.h deleted file mode 100644 index 3c02ae9f0a146c..00000000000000 --- a/be/src/storage/compaction/binlog_compaction.h +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "common/status.h" -#include "io/io_common.h" -#include "storage/compaction/compaction.h" - -namespace doris { - -class BinlogCompaction final : public CompactionMixin { -public: - BinlogCompaction(StorageEngine& engine, const TabletSharedPtr& tablet, int8_t compaction_level); - ~BinlogCompaction() override; - - Status prepare_compact() override; - Status execute_compact() override; - - std::optional profile_type() const override { - return CompactionProfileType::BINLOG; - } - - int8_t compaction_level() const override { return _compaction_level; } - -protected: - Status modify_rowsets() override; - - std::string_view compaction_name() const override { return "binlog compaction"; } - - ReaderType compaction_type() const override { return ReaderType::READER_BINLOG_COMPACTION; } - -private: - Status pick_rowsets_to_compact(); - - int8_t _compaction_level = -1; -}; - -} // namespace doris diff --git a/be/src/storage/compaction/binlog_compaction_policy.cpp b/be/src/storage/compaction/binlog_compaction_policy.cpp deleted file mode 100644 index 49832f758a172c..00000000000000 --- a/be/src/storage/compaction/binlog_compaction_policy.cpp +++ /dev/null @@ -1,235 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "storage/compaction/binlog_compaction_policy.h" - -#include - -#include "common/config.h" -#include "common/logging.h" -#include "storage/rowset/rowset.h" -#include "storage/tablet/tablet.h" -#include "storage/tablet/tablet_meta.h" -#include "util/time.h" - -namespace doris { - -int BinlogCompactionPolicy::pick_input_rowsets( - Tablet* tablet, const std::vector& candidate_rowsets, - int8_t compaction_level, std::vector* input_rowsets) const { - // 1) Filter rowsets by `compaction_level` - std::vector level_rowsets; - level_rowsets.reserve(candidate_rowsets.size()); - for (const auto& rs : candidate_rowsets) { - if (!rs->is_local() || rs->rowset_meta()->compaction_level() != compaction_level) { - continue; - } - if (!level_rowsets.empty() && - rs->start_version() != level_rowsets.back()->end_version() + 1) { - LOG(WARNING) << "rowset is non-continuous in the same compaction_level of binlog " - "compaction. tablet=" - << tablet->tablet_id() - << ", compaction_level=" << std::to_string(compaction_level) - << ", prev_version=" << level_rowsets.back()->version() - << ", next_version=" << rs->version(); - return 0; - } - level_rowsets.push_back(rs); - } - - // 2) Split `level_rowsets` into `full_enough_rowsets` and `remaining_rowsets`. - // - L0/L1: only physical rewrite. - // - LMax: Base([0-x]) + prefix ENOUGH rowsets are candidates for quick compact. - std::vector full_enough_rowsets; - std::vector remaining_rowsets; - full_enough_rowsets.reserve(level_rowsets.size()); - remaining_rowsets.reserve(level_rowsets.size()); - - bool find_base = false; - int full_enough_size = 0; - size_t idx = 0; - if (compaction_level == kBinlogCompactionMaxLevel - 1) { - if (!level_rowsets.empty() && level_rowsets[0]->start_version() == 0) { - find_base = true; - full_enough_rowsets.push_back(level_rowsets[0]); - ++full_enough_size; - idx = 1; - for (; idx < level_rowsets.size(); ++idx) { - const auto& rs = level_rowsets[idx]; - int64_t rs_score = rs->rowset_meta()->get_compaction_score(); - if (rs->data_disk_size() >= - config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || - rs_score >= config::binlog_compaction_file_count_threshold) { - full_enough_rowsets.push_back(rs); - ++full_enough_size; - continue; - } - break; - } - for (; idx < level_rowsets.size(); ++idx) { - remaining_rowsets.push_back(level_rowsets[idx]); - } - } else { - remaining_rowsets = level_rowsets; - } - } else { - remaining_rowsets = level_rowsets; - } - - // 3) Pick rowsets from `remaining_rowsets` (physical rewrite path). - std::vector picked_rowsets; - picked_rowsets.reserve(remaining_rowsets.size()); - int transient_size = 0; - int64_t total_size = 0; - int64_t compaction_score = 0; - const int64_t max_binlog_permits = (config::total_permits_for_compaction_score * - config::binlog_compaction_permits_percent + - 99) / - 100; - for (const auto& rs : remaining_rowsets) { - int64_t rs_score = rs->rowset_meta()->get_compaction_score(); - if (transient_size >= config::binlog_level_compaction_max_deltas || - compaction_score + rs_score > max_binlog_permits) { - break; - } - picked_rowsets.push_back(rs); - ++transient_size; - total_size += rs->data_disk_size(); - compaction_score += rs_score; - } - - // 4) Trigger check - // - L0/L1: only physical rewrite if trigger is met. - // - LMax: if physical rewrite trigger is NOT met, try quick compact; if both met, compare score. - bool can_do_binlog_compaction = false; - if (total_size >= config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || - compaction_score >= config::binlog_compaction_file_count_threshold) { - can_do_binlog_compaction = true; - } else if ((UnixMillis() - tablet->last_binlog_compaction_success_time(compaction_level)) / - 1000 >= - config::binlog_compaction_time_threshold_seconds) { - can_do_binlog_compaction = true; - } - - // 5) LMax quick compact vs physical rewrite. - if (compaction_level == kBinlogCompactionMaxLevel - 1 && find_base && full_enough_size > 1) { - int64_t quick_merge_score = 1; - for (int i = 1; i < full_enough_size; ++i) { - quick_merge_score += full_enough_rowsets[i]->rowset_meta()->get_compaction_score(); - } - - if (!can_do_binlog_compaction || quick_merge_score > compaction_score) { - input_rowsets->swap(full_enough_rowsets); - return full_enough_size; - } - } - - if (transient_size < 2) { - can_do_binlog_compaction = false; - } - - if (can_do_binlog_compaction) { - input_rowsets->swap(picked_rowsets); - return transient_size; - } - - input_rowsets->clear(); - return 0; -} - -uint32_t BinlogCompactionPolicy::calc_binlog_compaction_level_score(Tablet* tablet, - int8_t level) const { - uint32_t score = 0; - // Binlog tiered compaction score (L0..LMax) - // - // L0/L1/... : | rowsets ... | - // LMax : | Base([0-x]) | remaining rowsets ... | - // - // Base only performs meta-only merge (merge rowset meta), so get_compaction_score() - // treats Base score as 1. - for (const auto& [_, rs_meta] : tablet->tablet_meta()->all_row_binlog_rs_metas()) { - if (!rs_meta->is_local() || rs_meta->compaction_level() != level) { - continue; - } - score += rs_meta->get_compaction_score(); - } - return score; -} - -uint32_t BinlogCompactionPolicy::calc_binlog_compaction_score( - Tablet* tablet, int8_t* prefer_compaction_level) const { - uint32_t max_score = 0; - int8_t max_level = -1; - for (int8_t level = 0; level < kBinlogCompactionMaxLevel; ++level) { - uint32_t score = calc_binlog_compaction_level_score(tablet, level); - if (score > max_score) { - max_score = score; - max_level = level; - } - } - if (prefer_compaction_level != nullptr) { - *prefer_compaction_level = max_level; - } - return max_score; -} - -void BinlogCompactionPolicy::update_compaction_level( - Tablet* tablet, const std::vector& input_rowsets, - RowsetSharedPtr output_rowset) { - if (tablet->tablet_state() != TABLET_RUNNING || output_rowset->num_segments() == 0) { - return; - } - - int64_t first_level = 0; - for (size_t i = 0; i < input_rowsets.size(); ++i) { - int64_t cur_level = input_rowsets[i]->rowset_meta()->compaction_level(); - if (i == 0) { - first_level = cur_level; - continue; - } - - DCHECK_EQ(first_level, cur_level) - << "Compaction level mismatch, first_level: " << first_level - << ", cur_level: " << cur_level << ", tablet: " << tablet->tablet_id(); - if (first_level != cur_level) { - LOG(WARNING) << "Failed to check compaction level, first_level: " << first_level - << ", cur_level: " << cur_level << ", tablet: " << tablet->tablet_id(); - } - - DCHECK_EQ(input_rowsets[i]->start_version(), input_rowsets[i - 1]->end_version() + 1) - << "Binlog compaction input rowsets are non-continuous, prev_version: " - << input_rowsets[i - 1]->version() - << ", cur_version: " << input_rowsets[i]->version() - << ", tablet: " << tablet->tablet_id(); - if (input_rowsets[i]->start_version() != input_rowsets[i - 1]->end_version() + 1) { - LOG(WARNING) << "Failed to check binlog compaction input rowsets continuity, " - << "prev_version=" << input_rowsets[i - 1]->version() - << ", cur_version=" << input_rowsets[i]->version() - << ", tablet=" << tablet->tablet_id(); - } - } - - if (first_level == kBinlogCompactionMaxLevel - 1) { - // level max do not update compaction level - output_rowset->rowset_meta()->set_compaction_level(first_level); - return; - } - - output_rowset->rowset_meta()->set_compaction_level(first_level + 1); -} - -} // namespace doris diff --git a/be/src/storage/compaction/binlog_compaction_policy.h b/be/src/storage/compaction/binlog_compaction_policy.h deleted file mode 100644 index aaa75f38897308..00000000000000 --- a/be/src/storage/compaction/binlog_compaction_policy.h +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "storage/rowset/rowset_fwd.h" - -namespace doris { - -class Tablet; - -class BinlogCompactionPolicy { -public: - static constexpr int8_t kBinlogCompactionMaxLevel = 3; - - // Binlog compaction selection rules (tiered, L0..LMax) - // - // Score / Permits - // - For LMax, treat Base([0-x]) as score/permit=1, others use RowsetMeta::get_compaction_score(). - // - // Trigger (all levels): merge when ANY holds - // - size >= binlog_compaction_goal_size_mbytes * 1MB - // - score >= binlog_compaction_file_count_threshold - // - time >= binlog_compaction_time_threshold_seconds - // - // LMax "Base + `ENOUGH` + remaining" model (oldest -> newest) - // | Base([0-x]) | `ENOUGH` rowsets | remaining rowsets ... | - // `ENOUGH` is computed dynamically on LMax (not persisted): - // (rowset_size >= goal_size) OR (rowset_score >= file_count_threshold) - // - // Input Rowsets selection: - // - If physical rewrite trigger is NOT met: try quick compact first (requires Base([0-x])). - // - If both quick compact and physical rewrite are possible: compare score and pick the higher. - // - // Quick compact output must be OVERLAPPING. - int pick_input_rowsets(Tablet* tablet, const std::vector& candidate_rowsets, - int8_t compaction_level, - std::vector* input_rowsets) const; - - uint32_t calc_binlog_compaction_score(Tablet* tablet, int8_t* prefer_compaction_level) const; - uint32_t calc_binlog_compaction_level_score(Tablet* tablet, int8_t level) const; - - void update_compaction_level(Tablet* tablet, const std::vector& input_rowsets, - RowsetSharedPtr output_rowset); -}; - -} // namespace doris diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index f27f55d309661d..03c0862703aa4d 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -53,9 +53,9 @@ #include "io/io_common.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" -#include "storage/compaction/binlog_compaction_policy.h" #include "storage/compaction/collection_statistics.h" #include "storage/compaction/cumulative_compaction.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction_task_tracker.h" @@ -431,9 +431,7 @@ Status CompactionMixin::do_compact_ordered_rowsets() { RETURN_IF_ERROR(build_basic_info(true)); RowsetWriterContext ctx; RETURN_IF_ERROR(construct_output_rowset_writer(ctx)); - const auto& output_rowset_dir = compaction_type() == ReaderType::READER_BINLOG_COMPACTION - ? tablet()->row_binlog_path() - : tablet()->tablet_path(); + const auto& output_rowset_dir = tablet()->tablet_path(); LOG(INFO) << "start to do ordered data compaction, tablet=" << _tablet->tablet_id() << ", output_version=" << _output_version; @@ -472,8 +470,9 @@ Status CompactionMixin::do_compact_ordered_rowsets() { // bounds, so force aggregation on the output to keep the layout consistent // with `num_segments` / the aggregated flag, even if the config is off now. bool aggregate_key_bounds = - any_input_aggregated || (config::enable_aggregate_non_mow_key_bounds && - !_tablet->enable_unique_key_merge_on_write()); + any_input_aggregated || + (config::enable_aggregate_non_mow_key_bounds && + !_tablet->enable_unique_key_merge_on_write() && !tablet()->is_row_binlog_tablet()); rowset_meta->set_segments_key_bounds(segment_key_bounds, aggregate_key_bounds); rowset_meta->set_num_segment_rows(num_segment_rows); rowset_meta->set_commit_tso(commit_tso_range(_input_rowsets)); @@ -538,6 +537,9 @@ Status CompactionMixin::build_basic_info(bool is_ordered_compaction) { } bool CompactionMixin::handle_ordered_data_compaction() { + if (config::is_cloud_mode()) { + return false; + } if (!config::enable_ordered_data_compaction) { return false; } @@ -568,14 +570,14 @@ bool CompactionMixin::handle_ordered_data_compaction() { // The remote file system and full compaction does not support to link files. return false; } - bool is_binlog_compaction = compaction_type() == ReaderType::READER_BINLOG_COMPACTION; + bool is_binlog_compaction = _tablet->is_row_binlog_tablet(); if (is_binlog_compaction && !_input_rowsets.empty()) { RowsetIdUnorderedSet input_rowset_ids; input_rowset_ids.reserve(_input_rowsets.size()); for (const auto& rs : _input_rowsets) { input_rowset_ids.insert(rs->rowset_id()); } - if (_tablet->tablet_meta()->binlog_delvec().contain_rowsets(input_rowset_ids)) { + if (_tablet->tablet_meta()->delete_bitmap().contain_rowsets(input_rowset_ids)) { return false; } } @@ -603,15 +605,18 @@ bool CompactionMixin::handle_ordered_data_compaction() { } if (is_binlog_compaction) { + DCHECK(!_input_rowsets.empty()) << "tablet=" << _tablet->tablet_id(); + auto compaction_level = _input_rowsets.front()->rowset_meta()->compaction_level(); bool can_quick_merge_binlog = - compaction_level() == BinlogCompactionPolicy::kBinlogCompactionMaxLevel - 1 && + compaction_level == + BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1 && _input_rowsets.size() >= 2 && _input_rowsets[0]->start_version() == 0; if (!can_quick_merge_binlog) { return false; } // Binlog quick merge at LMax is a special meta/link compaction path selected by - // BinlogCompactionPolicy. It does not require the whole input to satisfy the normal + // BinlogCumulativeCompactionPolicy. It does not require the whole input to satisfy the normal // ordered-data tidy check, but the output rowset built by do_compact_ordered_rowsets() // is still NONOVERLAPPING. auto st = do_compact_ordered_rowsets(); @@ -725,9 +730,11 @@ Status CompactionMixin::execute_compact_impl(int64_t permits) { if (handle_ordered_data_compaction()) { _is_ordered_data_compaction = true; RETURN_IF_ERROR(modify_rowsets()); + int64_t input_compaction_level = _input_rowsets.front()->rowset_meta()->compaction_level(); LOG(INFO) << "succeed to do ordered data " << compaction_name() << ". tablet=" << _tablet->tablet_id() << ", output_version=" << _output_version << ", disk=" << tablet()->data_dir()->path() + << ", input_compaction_level=" << input_compaction_level << ", segments=" << _input_num_segments << ", input_row_num=" << _input_row_num << ", output_row_num=" << _output_rowset->num_rows() << ", input_rowsets_data_size=" << _input_rowsets_data_size @@ -752,17 +759,19 @@ Status CompactionMixin::execute_compact_impl(int64_t permits) { RETURN_IF_ERROR(merge_input_rowsets()); - // Currently, updates are only made in the time_series. + // Currently, updates are only made in the time_series and binlog policies. update_compaction_level(); RETURN_IF_ERROR(modify_rowsets()); auto* cumu_policy = tablet()->cumulative_compaction_policy(); DCHECK(cumu_policy); + int64_t input_compaction_level = _input_rowsets.front()->rowset_meta()->compaction_level(); LOG(INFO) << "succeed to do " << compaction_name() << " is_vertical=" << _is_vertical << ". tablet=" << _tablet->tablet_id() << ", output_version=" << _output_version << ", current_max_version=" << tablet()->max_version().second << ", disk=" << tablet()->data_dir()->path() + << ", input_compaction_level=" << input_compaction_level << ", input_segments=" << _input_num_segments << ", input_rowsets_data_size=" << PrettyPrinter::print_bytes(_input_rowsets_data_size) << ", input_rowsets_index_size=" @@ -1382,7 +1391,7 @@ Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) _tablet->keys_type() == KeysType::DUP_KEYS))) { construct_index_compaction_columns(ctx); } - if (compaction_type() == ReaderType::READER_BINLOG_COMPACTION) { + if (_tablet->is_row_binlog_tablet()) { ctx.write_binlog_opt().enable = true; } ctx.version = _output_version; @@ -1608,7 +1617,8 @@ bool CompactionMixin::_check_if_includes_input_rowsets( void CompactionMixin::update_compaction_level() { auto* cumu_policy = tablet()->cumulative_compaction_policy(); - if (cumu_policy && cumu_policy->name() == CUMULATIVE_TIME_SERIES_POLICY) { + if (cumu_policy && (cumu_policy->name() == CUMULATIVE_TIME_SERIES_POLICY || + cumu_policy->name() == CUMULATIVE_BINLOG_POLICY)) { int64_t compaction_level = cumu_policy->get_compaction_level(tablet(), _input_rowsets, _output_rowset); _output_rowset->rowset_meta()->set_compaction_level(compaction_level); @@ -1632,7 +1642,13 @@ Status Compaction::check_correctness() { int64_t CompactionMixin::get_compaction_permits() { int64_t permits = 0; + const int64_t point = tablet()->cumulative_layer_point(); for (auto&& rowset : _input_rowsets) { + if (tablet()->is_row_binlog_tablet() && point != Tablet::K_INVALID_CUMULATIVE_POINT && + rowset->end_version() < point) { + ++permits; + continue; + } permits += rowset->rowset_meta()->get_compaction_score(); } return permits; @@ -1733,7 +1749,7 @@ Status CloudCompactionMixin::execute_compact_impl(int64_t permits) { << _output_rowset->rowset_meta()->rowset_id().to_string(); }) - // Currently, updates are only made in the time_series. + // Currently, updates are only made in the time_series and binlog policies. update_compaction_level(); RETURN_IF_ERROR(_engine.meta_mgr().commit_rowset(*_output_rowset->rowset_meta().get(), _uuid, @@ -1944,6 +1960,9 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx.write_type = DataWriteType::TYPE_COMPACTION; ctx.compaction_type = compaction_type(); ctx.allow_packed_file = false; + if (_tablet->is_row_binlog_tablet()) { + ctx.write_binlog_opt().enable = true; + } // We presume that the data involved in cumulative compaction is sufficiently 'hot' // and should always be retained in the cache. @@ -2001,7 +2020,8 @@ void CloudCompactionMixin::update_compaction_level() { } else { auto compaction_policy = _tablet->tablet_meta()->compaction_policy(); auto cumu_policy = _engine.cumu_compaction_policy(compaction_policy); - if (cumu_policy && cumu_policy->name() == CUMULATIVE_TIME_SERIES_POLICY) { + if (cumu_policy && (cumu_policy->name() == CUMULATIVE_TIME_SERIES_POLICY || + cumu_policy->name() == std::string(CUMULATIVE_BINLOG_POLICY))) { int64_t compaction_level = cumu_policy->get_compaction_level( cloud_tablet(), _input_rowsets, _output_rowset); _output_rowset->rowset_meta()->set_compaction_level(compaction_level); diff --git a/be/src/storage/compaction/compaction.h b/be/src/storage/compaction/compaction.h index 9a9e598c2b1bf3..ed99581a3f77bd 100644 --- a/be/src/storage/compaction/compaction.h +++ b/be/src/storage/compaction/compaction.h @@ -76,7 +76,6 @@ class Compaction { virtual ReaderType compaction_type() const = 0; virtual std::string_view compaction_name() const = 0; - virtual int8_t compaction_level() const { return -1; } // Returns compaction profile type for task tracking. // Default returns std::nullopt (not tracked). Only base/cumulative/full override. diff --git a/be/src/storage/compaction/compaction_permit_limiter.cpp b/be/src/storage/compaction/compaction_permit_limiter.cpp index c1375265c41ac2..0d0852f5589974 100644 --- a/be/src/storage/compaction/compaction_permit_limiter.cpp +++ b/be/src/storage/compaction/compaction_permit_limiter.cpp @@ -22,25 +22,15 @@ namespace doris { -CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0), _binlog_used_permits(0) {} +CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} -bool CompactionPermitLimiter::try_request(int64_t permits, CompactionType compaction_type) { +bool CompactionPermitLimiter::try_request(int64_t permits) { const int64_t total_permits = config::total_permits_for_compaction_score; std::unique_lock lock(_permits_mutex); if (_used_permits + permits > total_permits) { return false; } - if (compaction_type == CompactionType::BINLOG_COMPACTION) { - const int64_t binlog_total_permits = - (total_permits * config::binlog_compaction_permits_percent + 99) / 100; - if (_binlog_used_permits + permits > binlog_total_permits) { - return false; - } - _binlog_used_permits += permits; - DorisMetrics::instance()->binlog_compaction_used_permits->set_value(_binlog_used_permits); - } - _used_permits += permits; DorisMetrics::instance()->compaction_used_permits->set_value(_used_permits); return true; @@ -76,12 +66,8 @@ void CompactionPermitLimiter::request(int64_t permits) { DorisMetrics::instance()->compaction_used_permits->set_value(_used_permits); } -void CompactionPermitLimiter::release(int64_t permits, CompactionType compaction_type) { +void CompactionPermitLimiter::release(int64_t permits) { std::unique_lock lock(_permits_mutex); - if (compaction_type == CompactionType::BINLOG_COMPACTION) { - _binlog_used_permits -= permits; - DorisMetrics::instance()->binlog_compaction_used_permits->set_value(_binlog_used_permits); - } _used_permits -= permits; _permits_cv.notify_one(); DorisMetrics::instance()->compaction_used_permits->set_value(_used_permits); diff --git a/be/src/storage/compaction/compaction_permit_limiter.h b/be/src/storage/compaction/compaction_permit_limiter.h index b94a51022772c9..c3403fb89b9448 100644 --- a/be/src/storage/compaction/compaction_permit_limiter.h +++ b/be/src/storage/compaction/compaction_permit_limiter.h @@ -40,18 +40,15 @@ class CompactionPermitLimiter { void request(int64_t permits); - bool try_request(int64_t permits, CompactionType compaction_type); + bool try_request(int64_t permits); - void release(int64_t permits, - CompactionType compaction_type = CompactionType::CUMULATIVE_COMPACTION); + void release(int64_t permits); int64_t usage() const { return _used_permits.load(std::memory_order_relaxed); } - int64_t binlog_usage() const { return _binlog_used_permits.load(std::memory_order_relaxed); } private: // sum of "permits" held by executing compaction tasks currently std::atomic _used_permits; - std::atomic _binlog_used_permits; std::mutex _permits_mutex; std::condition_variable _permits_cv; }; diff --git a/be/src/storage/compaction/cumulative_compaction.cpp b/be/src/storage/compaction/cumulative_compaction.cpp index 0c387f20661be6..c5ad4336d33974 100644 --- a/be/src/storage/compaction/cumulative_compaction.cpp +++ b/be/src/storage/compaction/cumulative_compaction.cpp @@ -177,13 +177,21 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { } } - int64_t max_score = config::cumulative_compaction_max_deltas; + int64_t max_score = tablet()->is_row_binlog_tablet() + ? config::binlog_level_compaction_max_deltas + : config::cumulative_compaction_max_deltas; int64_t process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage(); bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 8 / 10; if (tablet()->last_compaction_status.is() || memory_usage_high) { - max_score = std::max(config::cumulative_compaction_max_deltas / - config::cumulative_compaction_max_deltas_factor, - config::cumulative_compaction_min_deltas + 1); + if (tablet()->is_row_binlog_tablet()) { + max_score = std::max(config::binlog_level_compaction_max_deltas / + config::cumulative_compaction_max_deltas_factor, + 1); + } else { + max_score = std::max(config::cumulative_compaction_max_deltas / + config::cumulative_compaction_max_deltas_factor, + config::cumulative_compaction_min_deltas + 1); + } } size_t compaction_score = 0; diff --git a/be/src/storage/compaction/cumulative_compaction_binlog_policy.cpp b/be/src/storage/compaction/cumulative_compaction_binlog_policy.cpp new file mode 100644 index 00000000000000..e078a6b8033d09 --- /dev/null +++ b/be/src/storage/compaction/cumulative_compaction_binlog_policy.cpp @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/compaction/cumulative_compaction_binlog_policy.h" + +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" +#include "util/time.h" + +namespace doris { + +bool BinlogCumulativeCompactionPolicy::is_compaction_enough( + const RowsetMetaSharedPtr& rowset_meta) const { + if (rowset_meta->compaction_level() != kBinlogCompactionMaxLevel - 1) { + return false; + } + if (rowset_meta->start_version() == 0) { + return true; + } + return rowset_meta->data_disk_size() >= + config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || + rowset_meta->get_compaction_score() >= config::binlog_compaction_file_count_threshold; +} + +void BinlogCumulativeCompactionPolicy::calculate_cumulative_point( + Tablet* tablet, const RowsetMetaMapContainer& all_rowsets, int64_t current_cumulative_point, + int64_t* cumulative_point) { + *cumulative_point = Tablet::K_INVALID_CUMULATIVE_POINT; + if (current_cumulative_point != Tablet::K_INVALID_CUMULATIVE_POINT || all_rowsets.empty()) { + return; + } + + std::vector rowset_metas; + rowset_metas.reserve(all_rowsets.size()); + for (const auto& [_, rs_meta] : all_rowsets) { + if (rs_meta->is_local()) { + rowset_metas.emplace_back(rs_meta); + } + } + std::sort(rowset_metas.begin(), rowset_metas.end(), RowsetMeta::comparator); + + int64_t prev_version = -1; + for (const auto& rs_meta : rowset_metas) { + if (*cumulative_point == Tablet::K_INVALID_CUMULATIVE_POINT) { + *cumulative_point = rs_meta->start_version(); + } + if (rs_meta->start_version() > prev_version + 1 || !is_compaction_enough(rs_meta)) { + break; + } + prev_version = rs_meta->end_version(); + *cumulative_point = prev_version + 1; + } + + VLOG_NOTICE << "binlog compaction policy, calculate cumulative point value=" + << *cumulative_point << ", tablet=" << tablet->tablet_id(); +} + +void BinlogCumulativeCompactionPolicy::update_cumulative_point( + Tablet* tablet, const std::vector& input_rowsets, + RowsetSharedPtr output_rowset, Version& last_delete_version) { + if (tablet->tablet_state() != TABLET_RUNNING || input_rowsets.empty() || + output_rowset->num_segments() == 0 || + output_rowset->rowset_meta()->compaction_level() != kBinlogCompactionMaxLevel - 1 || + !is_compaction_enough(output_rowset->rowset_meta())) { + return; + } + + const int64_t point = tablet->cumulative_layer_point(); + if (point == Tablet::K_INVALID_CUMULATIVE_POINT || output_rowset->start_version() > point) { + return; + } + tablet->set_cumulative_layer_point(output_rowset->end_version() + 1); +} + +int BinlogCumulativeCompactionPolicy::pick_input_rowsets( + Tablet* tablet, const std::vector& candidate_rowsets, + int8_t compaction_level, int64_t max_compaction_score, + std::vector* input_rowsets) const { + // 1) Filter rowsets by `compaction_level` + std::vector level_rowsets; + level_rowsets.reserve(candidate_rowsets.size()); + for (const auto& rs : candidate_rowsets) { + if (!rs->is_local() || rs->rowset_meta()->compaction_level() != compaction_level) { + continue; + } + if (!level_rowsets.empty() && + rs->start_version() != level_rowsets.back()->end_version() + 1) { + LOG(WARNING) << "rowset is non-continuous in the same compaction_level of binlog " + "compaction. tablet=" + << tablet->tablet_id() + << ", compaction_level=" << std::to_string(compaction_level) + << ", prev_version=" << level_rowsets.back()->version() + << ", next_version=" << rs->version(); + return 0; + } + level_rowsets.push_back(rs); + } + + // 2) Split `level_rowsets` into `compact_enough_rowsets` and `remaining_rowsets`. + // - L0/L1: only physical rewrite. + // - LMax: rowsets before cumulative point are candidates for quick compact. + std::vector compact_enough_rowsets; + std::vector remaining_rowsets; + compact_enough_rowsets.reserve(level_rowsets.size()); + remaining_rowsets.reserve(level_rowsets.size()); + + int compact_enough_size = 0; + const int64_t point = tablet->cumulative_layer_point(); + if (compaction_level == kBinlogCompactionMaxLevel - 1 && + point != Tablet::K_INVALID_CUMULATIVE_POINT) { + for (const auto& rs : level_rowsets) { + if (rs->end_version() < point) { + compact_enough_rowsets.push_back(rs); + ++compact_enough_size; + continue; + } + remaining_rowsets.push_back(rs); + } + } else { + remaining_rowsets = level_rowsets; + } + + // 3) Pick rowsets from `remaining_rowsets` (physical rewrite path). + std::vector picked_rowsets; + picked_rowsets.reserve(remaining_rowsets.size()); + int transient_size = 0; + int64_t total_size = 0; + int64_t compaction_score = 0; + for (const auto& rs : remaining_rowsets) { + int64_t rs_score = rs->rowset_meta()->get_compaction_score(); + if (transient_size >= max_compaction_score) { + break; + } + picked_rowsets.push_back(rs); + ++transient_size; + total_size += rs->data_disk_size(); + compaction_score += rs_score; + } + + // 4) Trigger check + // - L0/L1: only physical rewrite if trigger is met. + // - LMax: if physical rewrite trigger is NOT met, try quick compact; if both met, compare score. + bool can_do_binlog_compaction = false; + if (total_size >= config::binlog_compaction_goal_size_mbytes * 1024 * 1024 || + compaction_score >= config::binlog_compaction_file_count_threshold) { + can_do_binlog_compaction = true; + } else if ((UnixMillis() - tablet->last_cumu_compaction_success_time()) / 1000 >= + config::binlog_compaction_time_threshold_seconds) { + can_do_binlog_compaction = true; + } + + // 5) LMax quick compact vs physical rewrite. + if (compaction_level == kBinlogCompactionMaxLevel - 1 && compact_enough_size > 1) { + int64_t quick_merge_score = compact_enough_size; + + if (!can_do_binlog_compaction || quick_merge_score > compaction_score) { + input_rowsets->swap(compact_enough_rowsets); + return compact_enough_size; + } + } + + if (transient_size < 2) { + can_do_binlog_compaction = false; + } + + if (can_do_binlog_compaction) { + input_rowsets->swap(picked_rowsets); + return transient_size; + } + + input_rowsets->clear(); + return 0; +} + +uint32_t BinlogCumulativeCompactionPolicy::calc_binlog_compaction_level_score(Tablet* tablet, + int8_t level) const { + uint32_t score = 0; + const int64_t point = tablet->cumulative_layer_point(); + // Binlog tiered compaction score (L0..LMax) + // + // L0/L1/... : | rowsets ... | + // + // LMax: + // version: 0 point + // |------------------------------------|------------------- ... + // rowsets: | compact enough | compact enough | compacting rowsets ... + // score : | 1 | 1 | get_compaction_score() + // + // Each rowset before cumulative point is compact-complete, and its score is treated as 1. + for (const auto& [_, rs_meta] : tablet->tablet_meta()->all_rs_metas()) { + if (!rs_meta->is_local() || rs_meta->compaction_level() != level) { + continue; + } + if (level == kBinlogCompactionMaxLevel - 1 && point != Tablet::K_INVALID_CUMULATIVE_POINT && + rs_meta->end_version() < point) { + score += 1; + continue; + } + score += rs_meta->get_compaction_score(); + } + return score; +} + +uint32_t BinlogCumulativeCompactionPolicy::calc_binlog_compaction_level_score( + Tablet* tablet, const std::vector& candidate_rowsets, int8_t level) const { + uint32_t score = 0; + const int64_t point = tablet->cumulative_layer_point(); + for (const auto& rs : candidate_rowsets) { + const auto& rs_meta = rs->rowset_meta(); + if (!rs->is_local() || rs_meta->compaction_level() != level) { + continue; + } + if (level == kBinlogCompactionMaxLevel - 1 && point != Tablet::K_INVALID_CUMULATIVE_POINT && + rs_meta->end_version() < point) { + score += 1; + continue; + } + score += rs_meta->get_compaction_score(); + } + return score; +} + +uint32_t BinlogCumulativeCompactionPolicy::calc_binlog_compaction_score( + Tablet* tablet, int8_t* prefer_compaction_level) const { + uint32_t max_score = 0; + int8_t max_level = -1; + for (int8_t level = 0; level < kBinlogCompactionMaxLevel; ++level) { + uint32_t score = calc_binlog_compaction_level_score(tablet, level); + if (score > max_score) { + max_score = score; + max_level = level; + } + } + if (prefer_compaction_level != nullptr) { + *prefer_compaction_level = max_level; + } + return max_score; +} + +uint32_t BinlogCumulativeCompactionPolicy::calc_binlog_compaction_score( + Tablet* tablet, const std::vector& candidate_rowsets, + int8_t* prefer_compaction_level) const { + uint32_t max_score = 0; + int8_t max_level = -1; + for (int8_t level = 0; level < kBinlogCompactionMaxLevel; ++level) { + uint32_t score = calc_binlog_compaction_level_score(tablet, candidate_rowsets, level); + if (score > max_score) { + max_score = score; + max_level = level; + } + } + if (prefer_compaction_level != nullptr) { + *prefer_compaction_level = max_level; + } + return max_score; +} + +int BinlogCumulativeCompactionPolicy::pick_input_rowsets( + Tablet* tablet, const std::vector& candidate_rowsets, + const int64_t max_compaction_score, const int64_t min_compaction_score, + std::vector* input_rowsets, Version* last_delete_version, + size_t* compaction_score, bool allow_delete) { + int8_t compaction_level = -1; + calc_binlog_compaction_score(tablet, candidate_rowsets, &compaction_level); + if (compaction_level < 0) { + *compaction_score = 0; + return 0; + } + + int picked_score = pick_input_rowsets(tablet, candidate_rowsets, compaction_level, + max_compaction_score, input_rowsets); + *compaction_score = picked_score; + return picked_score; +} + +uint32_t BinlogCumulativeCompactionPolicy::calc_cumulative_compaction_score(Tablet* tablet) { + return calc_binlog_compaction_score(tablet); +} + +int64_t BinlogCumulativeCompactionPolicy::get_compaction_level( + Tablet* tablet, const std::vector& input_rowsets, + RowsetSharedPtr output_rowset) { + DCHECK(!input_rowsets.empty()) << "tablet=" << tablet->tablet_id(); + int64_t first_level = input_rowsets.front()->rowset_meta()->compaction_level(); + for (size_t i = 1; i < input_rowsets.size(); ++i) { + int64_t cur_level = input_rowsets[i]->rowset_meta()->compaction_level(); + DCHECK_EQ(first_level, cur_level) + << "Compaction level mismatch, first_level: " << first_level + << ", cur_level: " << cur_level << ", tablet: " << tablet->tablet_id(); + if (first_level != cur_level) { + LOG(WARNING) << "Failed to check compaction level, first_level: " << first_level + << ", cur_level: " << cur_level << ", tablet: " << tablet->tablet_id(); + } + + DCHECK_EQ(input_rowsets[i]->start_version(), input_rowsets[i - 1]->end_version() + 1) + << "Binlog compaction input rowsets are non-continuous, prev_version: " + << input_rowsets[i - 1]->version() + << ", cur_version: " << input_rowsets[i]->version() + << ", tablet: " << tablet->tablet_id(); + if (input_rowsets[i]->start_version() != input_rowsets[i - 1]->end_version() + 1) { + LOG(WARNING) << "Failed to check binlog compaction input rowsets continuity, " + << "prev_version=" << input_rowsets[i - 1]->version() + << ", cur_version=" << input_rowsets[i]->version() + << ", tablet=" << tablet->tablet_id(); + } + } + + if (first_level == kBinlogCompactionMaxLevel - 1) { + return first_level; + } + return first_level + 1; +} + +} // namespace doris diff --git a/be/src/storage/compaction/cumulative_compaction_binlog_policy.h b/be/src/storage/compaction/cumulative_compaction_binlog_policy.h new file mode 100644 index 00000000000000..0503cff7f773f1 --- /dev/null +++ b/be/src/storage/compaction/cumulative_compaction_binlog_policy.h @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/compaction/cumulative_compaction_policy.h" +#include "storage/rowset/rowset_fwd.h" + +namespace doris { + +class Tablet; + +inline constexpr std::string_view CUMULATIVE_BINLOG_POLICY = "binlog"; + +class BinlogCumulativeCompactionPolicy final : public CumulativeCompactionPolicy { +public: + BinlogCumulativeCompactionPolicy() = default; + ~BinlogCumulativeCompactionPolicy() override = default; + + static constexpr int8_t kBinlogCompactionMaxLevel = 3; + + // Binlog compaction selection rules (tiered, L0..LMax) + // + // Score / Permits + // - L0/L1 use RowsetMeta::get_compaction_score(). + // - For LMax, each rowset before cumulative point is score/permit=1, + // others use RowsetMeta::get_compaction_score(). + // + // Trigger (all levels): merge when ANY holds + // - size >= binlog_compaction_goal_size_mbytes * 1MB + // - score >= binlog_compaction_file_count_threshold + // - time >= binlog_compaction_time_threshold_seconds + // + // LMax model (oldest -> newest): + // version: 0 point + // |------------------------------------|------------------- ... + // rowsets: | compact enough | compact enough | compacting rowsets ... + // score : | 1 | 1 | get_compaction_score() + // + // Input Rowsets selection: + // - If physical rewrite trigger is NOT met: try quick compact first. + // - If both quick compact and physical rewrite are possible: compare score and pick the higher. + // + // Quick compact output must be OVERLAPPING. + int pick_input_rowsets(Tablet* tablet, const std::vector& candidate_rowsets, + int8_t compaction_level, int64_t max_compaction_score, + std::vector* input_rowsets) const; + + uint32_t calc_binlog_compaction_score(Tablet* tablet, + int8_t* prefer_compaction_level = nullptr) const; + uint32_t calc_binlog_compaction_score(Tablet* tablet, + const std::vector& candidate_rowsets, + int8_t* prefer_compaction_level) const; + uint32_t calc_binlog_compaction_level_score(Tablet* tablet, int8_t level) const; + uint32_t calc_binlog_compaction_level_score( + Tablet* tablet, const std::vector& candidate_rowsets, + int8_t level) const; + + bool is_compaction_enough(const RowsetMetaSharedPtr& rowset_meta) const; + + void calculate_cumulative_point(Tablet* tablet, const RowsetMetaMapContainer& all_rowsets, + int64_t current_cumulative_point, + int64_t* cumulative_point) override; + + int pick_input_rowsets(Tablet* tablet, const std::vector& candidate_rowsets, + const int64_t max_compaction_score, const int64_t min_compaction_score, + std::vector* input_rowsets, + Version* last_delete_version, size_t* compaction_score, + bool allow_delete = false) override; + + void update_cumulative_point(Tablet* tablet, const std::vector& input_rowsets, + RowsetSharedPtr output_rowset, + Version& last_delete_version) override; + + uint32_t calc_cumulative_compaction_score(Tablet* tablet) override; + + int64_t get_compaction_level(Tablet* tablet, const std::vector& input_rowsets, + RowsetSharedPtr output_rowset) override; + + std::string_view name() override { return CUMULATIVE_BINLOG_POLICY; } +}; + +} // namespace doris diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp b/be/src/storage/compaction/cumulative_compaction_policy.cpp index a120d941e6f410..a34b5b8ae5e477 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp @@ -25,8 +25,11 @@ #include "common/config.h" #include "common/logging.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/olap_common.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_meta.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_meta.h" #include "util/debug_points.h" @@ -424,6 +427,8 @@ CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( return std::make_shared(); } else if (compaction_policy == CUMULATIVE_SIZE_BASED_POLICY) { return std::make_shared(); + } else if (compaction_policy == CUMULATIVE_BINLOG_POLICY) { + return std::make_shared(); } return std::make_shared(); } diff --git a/be/src/storage/compaction/cumulative_compaction_policy.h b/be/src/storage/compaction/cumulative_compaction_policy.h index f6c01b1be98c99..45ea6265c1087a 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.h +++ b/be/src/storage/compaction/cumulative_compaction_policy.h @@ -25,12 +25,12 @@ #include #include "common/config.h" -#include "storage/rowset/rowset.h" -#include "storage/rowset/rowset_meta.h" +#include "storage/rowset/rowset_fwd.h" namespace doris { class Tablet; +class BaseTablet; struct Version; inline constexpr std::string_view CUMULATIVE_SIZE_BASED_POLICY = "size_based"; diff --git a/be/src/storage/compaction/cumulative_compaction_time_series_policy.cpp b/be/src/storage/compaction/cumulative_compaction_time_series_policy.cpp index af023f6becab10..884bec31fcf74c 100644 --- a/be/src/storage/compaction/cumulative_compaction_time_series_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_time_series_policy.cpp @@ -481,4 +481,4 @@ int64_t TimeSeriesCumulativeCompactionPolicy::get_compaction_level( return first_level + 1; } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/compaction/cumulative_compaction_time_series_policy.h b/be/src/storage/compaction/cumulative_compaction_time_series_policy.h index 839a2ea407ef6e..2e937fe7885d6d 100644 --- a/be/src/storage/compaction/cumulative_compaction_time_series_policy.h +++ b/be/src/storage/compaction/cumulative_compaction_time_series_policy.h @@ -79,4 +79,4 @@ class TimeSeriesCumulativeCompactionPolicy final : public CumulativeCompactionPo RowsetSharedPtr output_rowset); }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/compaction_task_tracker.cpp b/be/src/storage/compaction_task_tracker.cpp index 98a5bae962cd9f..afac74b9246642 100644 --- a/be/src/storage/compaction_task_tracker.cpp +++ b/be/src/storage/compaction_task_tracker.cpp @@ -30,8 +30,6 @@ const char* to_string(CompactionProfileType type) { return "cumulative"; case CompactionProfileType::FULL: return "full"; - case CompactionProfileType::BINLOG: - return "binlog"; } return "unknown"; } diff --git a/be/src/storage/compaction_task_tracker.h b/be/src/storage/compaction_task_tracker.h index 37f3679f38bc96..1b5ce7e8fed411 100644 --- a/be/src/storage/compaction_task_tracker.h +++ b/be/src/storage/compaction_task_tracker.h @@ -32,7 +32,6 @@ enum class CompactionProfileType : uint8_t { BASE = 0, CUMULATIVE = 1, FULL = 2, - BINLOG = 3, }; const char* to_string(CompactionProfileType type); diff --git a/be/src/storage/data_dir.cpp b/be/src/storage/data_dir.cpp index ab7b56f610a724..a7998db755c781 100644 --- a/be/src/storage/data_dir.cpp +++ b/be/src/storage/data_dir.cpp @@ -519,36 +519,14 @@ Status DataDir::load() { rowset_partition_id_eq_0_num, config::ignore_invalid_partition_id_rowset_num)); } - std::map> rowset_id_to_row_binlog_metas; - int64_t row_binlog_cnt {0}; - int64_t invalid_row_binlog_cnt {0}; - auto load_row_binlog_meta_func = - [&rowset_id_to_row_binlog_metas, &row_binlog_cnt, &invalid_row_binlog_cnt]( - const TabletUid& tablet_uid, const RowsetId& rowset_id, - const RowsetId& row_binlog_rowset_id, const std::string& val) -> bool { - RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); - bool parsed = rowset_meta->init(val); - if (!parsed) { - LOG(WARNING) << "parse binlog meta string failed, tablet_uid=" << tablet_uid - << ", rowset_id=" << rowset_id - << ", row_binlog_rowset_id=" << row_binlog_rowset_id; - ++invalid_row_binlog_cnt; - return true; + // Row binlog rowset is now a normal rowset under its own binlog tablet, loaded above. + // Index them by txn id so each base rowset can re-attach its paired binlog rowset on recovery. + std::map txn_id_to_row_binlog_meta; + for (auto&& rowset_meta : dir_rowset_metas) { + if (rowset_meta->is_row_binlog()) { + txn_id_to_row_binlog_meta[rowset_meta->txn_id()] = rowset_meta; } - DCHECK(rowset_meta->is_row_binlog()); - DCHECK_EQ(rowset_meta->tablet_uid(), tablet_uid); - rowset_id_to_row_binlog_metas[rowset_id].emplace_back(std::move(rowset_meta)); - ++row_binlog_cnt; - return true; - }; - - MonotonicStopWatch rb_timer; - rb_timer.start(); - RETURN_IF_ERROR(RowsetMetaManager::traverse_row_binlog_metas(_meta, load_row_binlog_meta_func)); - rb_timer.stop(); - LOG(INFO) << "load binlog meta finished, cost: " << rb_timer.elapsed_time_milliseconds() - << " ms, data dir: " << _path << ", count: " << row_binlog_cnt - << ", invalid: " << invalid_row_binlog_cnt; + } // traverse rowset // 1. add committed rowset to txn map @@ -573,35 +551,41 @@ Status DataDir::load() { continue; } - std::vector attach_rowsets; - std::map attach_rowset_map; - bool invalid_attach_rowset = false; - if (auto it = rowset_id_to_row_binlog_metas.find(rowset_meta->rowset_id()); - it != rowset_id_to_row_binlog_metas.end()) { - for (auto& attach_rowset_meta : it->second) { - DCHECK_EQ(attach_rowset_meta->rowset_state(), rowset_meta->rowset_state()); - if (!attach_rowset_meta->tablet_schema()) { - attach_rowset_meta->set_tablet_schema(tablet->row_binlog_tablet_schema()); - } + // Committed row binlog rowset is recovered with base rowset. + if (rowset_meta->is_row_binlog() && + rowset_meta->rowset_state() == RowsetStatePB::COMMITTED) { + continue; + } - RowsetSharedPtr attach_rowset; - Status attach_create_status = - tablet->create_rowset(attach_rowset_meta, &attach_rowset); - if (!attach_create_status.ok()) { - LOG(WARNING) << "could not create rowset from binlog rowset meta: " - << " rowset_id: " << attach_rowset_meta->rowset_id() - << " rowset_type: " << attach_rowset_meta->rowset_type() - << " rowset_state: " << attach_rowset_meta->rowset_state(); - invalid_attach_rowset = true; - break; - } - attach_rowset_map.emplace(attach_rowset_meta->rowset_id(), - attach_rowset_meta->get_rowset_pb()); - attach_rowsets.emplace_back(std::move(attach_rowset)); + RowBinlogTxnInfo attach_row_binlog; + if (auto it = txn_id_to_row_binlog_meta.find(rowset_meta->txn_id()); + it != txn_id_to_row_binlog_meta.end()) { + const RowsetMetaSharedPtr& attach_row_binlog_rowset_meta = it->second; + DCHECK_EQ(attach_row_binlog_rowset_meta->rowset_state(), rowset_meta->rowset_state()); + TabletSharedPtr binlog_tablet = _engine.tablet_manager()->get_tablet( + attach_row_binlog_rowset_meta->tablet_id()); + if (binlog_tablet == nullptr) { + LOG(WARNING) << "could not find binlog tablet: " + << attach_row_binlog_rowset_meta->tablet_id() + << " for binlog rowset: " + << attach_row_binlog_rowset_meta->rowset_id(); + ++invalid_rowset_counter; + continue; } - } - if (invalid_attach_rowset) { - continue; + if (!attach_row_binlog_rowset_meta->tablet_schema()) { + attach_row_binlog_rowset_meta->set_tablet_schema(binlog_tablet->tablet_schema()); + } + Status attach_create_status = binlog_tablet->create_rowset( + attach_row_binlog_rowset_meta, &attach_row_binlog.rowset); + if (!attach_create_status.ok()) { + LOG(WARNING) << "could not create rowset from binlog rowset meta: " + << " rowset_id: " << attach_row_binlog_rowset_meta->rowset_id() + << " rowset_type: " << attach_row_binlog_rowset_meta->rowset_type() + << " rowset_state: " << attach_row_binlog_rowset_meta->rowset_state(); + ++invalid_rowset_counter; + continue; + } + attach_row_binlog.tablet = std::move(binlog_tablet); } RowsetSharedPtr rowset; @@ -615,78 +599,74 @@ Status DataDir::load() { } std::optional binlog_format = std::nullopt; - const std::map* attach_rowset_map_ptr = nullptr; - if (!attach_rowset_map.empty()) { + std::optional attach_row_binlog_rowset_meta; + if (attach_row_binlog.rowset != nullptr) { binlog_format = BinlogFormatPB::ROW; - attach_rowset_map_ptr = &attach_rowset_map; + attach_row_binlog_rowset_meta = + attach_row_binlog.rowset->rowset_meta()->get_rowset_pb(); } + std::string attach_binlog_rowset_id = + attach_row_binlog.rowset != nullptr + ? attach_row_binlog.rowset->rowset_id().to_string() + : "0"; + if (rowset_meta->rowset_state() == RowsetStatePB::COMMITTED && rowset_meta->tablet_uid() == tablet->tablet_uid()) { if (!rowset_meta->tablet_schema()) { rowset_meta->set_tablet_schema(tablet->tablet_schema()); - RETURN_IF_ERROR(RowsetMetaManager::save( - _meta, rowset_meta->tablet_uid(), rowset_meta->rowset_id(), - rowset_meta->get_rowset_pb(), binlog_format, attach_rowset_map_ptr)); + RETURN_IF_ERROR(RowsetMetaManager::save(_meta, rowset_meta->tablet_uid(), + rowset_meta->rowset_id(), + rowset_meta->get_rowset_pb(), binlog_format, + attach_row_binlog_rowset_meta)); } std::vector rowset_ids {rowset_meta->rowset_id()}; - for (const auto& attach_rowset : attach_rowsets) { - if (attach_rowset != nullptr) { - rowset_ids.emplace_back(attach_rowset->rowset_id()); - } + if (attach_row_binlog.rowset != nullptr) { + rowset_ids.emplace_back(attach_row_binlog.rowset->rowset_id()); } Status commit_txn_status = _engine.txn_manager()->commit_txn( _meta, rowset_meta->partition_id(), rowset_meta->txn_id(), rowset_meta->tablet_id(), rowset_meta->tablet_uid(), rowset_meta->load_id(), rowset, _engine.pending_local_rowsets().add(rowset_ids), true, nullptr, - attach_rowsets.empty() ? nullptr : &attach_rowsets); + attach_row_binlog); if (commit_txn_status || commit_txn_status.is()) { LOG(INFO) << "successfully to add committed rowset: " << rowset_meta->rowset_id() << " to tablet: " << rowset_meta->tablet_id() << " schema hash: " << rowset_meta->tablet_schema_hash() - << " for txn: " << rowset_meta->txn_id() << ", binlog rowset: " - << (attach_rowsets.empty() ? "0" - : attach_rowsets[0]->rowset_id().to_string()); + << " for txn: " << rowset_meta->txn_id() + << ", binlog rowset: " << attach_binlog_rowset_id; } else if (commit_txn_status.is()) { LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id() << " to tablet: " << rowset_meta->tablet_id() << " for txn: " << rowset_meta->txn_id() - << " error: " << commit_txn_status << ", binlog rowset: " - << (attach_rowsets.empty() - ? "0" - : attach_rowsets[0]->rowset_id().to_string()); + << " error: " << commit_txn_status + << ", binlog rowset: " << attach_binlog_rowset_id; return commit_txn_status; } else { LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id() << " to tablet: " << rowset_meta->tablet_id() << " for txn: " << rowset_meta->txn_id() - << " error: " << commit_txn_status << ", binlog rowset: " - << (attach_rowsets.empty() - ? "0" - : attach_rowsets[0]->rowset_id().to_string()); + << " error: " << commit_txn_status + << ", binlog rowset: " << attach_binlog_rowset_id; } } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && rowset_meta->tablet_uid() == tablet->tablet_uid()) { if (!rowset_meta->tablet_schema()) { rowset_meta->set_tablet_schema(tablet->tablet_schema()); - RETURN_IF_ERROR(RowsetMetaManager::save( - _meta, rowset_meta->tablet_uid(), rowset_meta->rowset_id(), - rowset_meta->get_rowset_pb(), binlog_format, attach_rowset_map_ptr)); + RETURN_IF_ERROR(RowsetMetaManager::save(_meta, rowset_meta->tablet_uid(), + rowset_meta->rowset_id(), + rowset_meta->get_rowset_pb(), binlog_format, + attach_row_binlog_rowset_meta)); } - DCHECK_LE(attach_rowsets.size(), 1); - Status publish_status = tablet->add_rowset( - rowset, attach_rowsets.empty() ? nullptr : attach_rowsets[0]); + Status publish_status = tablet->add_rowset(rowset); if (!publish_status && !publish_status.is()) { LOG(WARNING) << "add visible rowset to tablet failed rowset_id:" << rowset->rowset_id() << " tablet id: " << rowset_meta->tablet_id() << " txn id:" << rowset_meta->txn_id() << " start_version: " << rowset_meta->version().first << " end_version: " << rowset_meta->version().second - << ", binlog rowset: " - << (attach_rowsets.empty() - ? "0" - : attach_rowsets[0]->rowset_id().to_string()); + << ", binlog rowset: " << attach_binlog_rowset_id; } } else { LOG(WARNING) << "find invalid rowset: " << rowset_meta->rowset_id() @@ -695,9 +675,7 @@ Status DataDir::load() { << " schema hash: " << rowset_meta->tablet_schema_hash() << " txn: " << rowset_meta->txn_id() << " current valid tablet uid: " << tablet->tablet_uid() - << ", binlog rowset: " - << (attach_rowsets.empty() ? "0" - : attach_rowsets[0]->rowset_id().to_string()); + << ", binlog rowset: " << attach_binlog_rowset_id; ++invalid_rowset_counter; } } @@ -711,13 +689,8 @@ Status DataDir::load() { if (!tablet) { return true; } - const auto& all_data_rowsets = tablet->tablet_meta()->all_rs_metas(); - const auto& all_row_binlogs = tablet->tablet_meta()->all_row_binlog_rs_metas(); RowsetIdUnorderedSet rowset_ids; - for (const auto& [_, rowset_meta] : all_data_rowsets) { - rowset_ids.insert(rowset_meta->rowset_id()); - } - for (auto& [_, rowset_meta] : all_row_binlogs) { + for (const auto& [_, rowset_meta] : tablet->tablet_meta()->all_rs_metas()) { rowset_ids.insert(rowset_meta->rowset_id()); } @@ -726,14 +699,12 @@ Status DataDir::load() { int rst_ids_size = delete_bitmap_pb.rowset_ids_size(); int seg_ids_size = delete_bitmap_pb.segment_ids_size(); int seg_maps_size = delete_bitmap_pb.segment_delete_bitmaps_size(); - int binlog_mark_size = delete_bitmap_pb.is_binlog_delvec_size(); CHECK(rst_ids_size == seg_ids_size && seg_ids_size == seg_maps_size); - CHECK(binlog_mark_size == 0 || binlog_mark_size == rst_ids_size); for (int i = 0; i < rst_ids_size; ++i) { RowsetId rst_id; rst_id.init(delete_bitmap_pb.rowset_ids(i)); - // only process the rowset in _rs_metas and _row_binlog_rs_metas + // only process rowsets in current tablet meta. if (rowset_ids.find(rst_id) == rowset_ids.end()) { ++unknown_dbm_cnt; continue; @@ -748,19 +719,11 @@ Status DataDir::load() { } auto bitmap = delete_bitmap_pb.segment_delete_bitmaps(i).data(); - bool from_binlog = delete_bitmap_pb.is_binlog_delvec_size() > 0 - ? delete_bitmap_pb.is_binlog_delvec(i) - : false; - if (!from_binlog) { - tablet->tablet_meta()->delete_bitmap().delete_bitmap[{rst_id, seg_id, version}] = - roaring::Roaring::read(bitmap); - } else { - tablet->tablet_meta()->binlog_delvec().delete_bitmap[{rst_id, seg_id, version}] = - roaring::Roaring::read(bitmap); - } + tablet->tablet_meta()->delete_bitmap().delete_bitmap[{rst_id, seg_id, version}] = + roaring::Roaring::read(bitmap); VLOG_ROW << "successfully to add delete_bitmap, tablet_id=" << tablet->tablet_id() - << ", rowset_id=" << rst_id << ", seg_id=" << seg_id << ", version=" << version - << ", from_binlog=" << from_binlog; + << ", rowset_id=" << rst_id << ", seg_id=" << seg_id + << ", version=" << version; } return true; }; @@ -870,9 +833,6 @@ void DataDir::_perform_rowset_gc(const std::string& tablet_schema_hash_path) { tablet->traverse_rowsets( [&rowsets_in_version_map](auto& rs) { rowsets_in_version_map.insert(rs->rowset_id()); }, true); - for (const auto& [_, rb_meta] : tablet->tablet_meta()->all_row_binlog_rs_metas()) { - rowsets_in_version_map.insert(rb_meta->rowset_id()); - } DBUG_EXECUTE_IF("DataDir::_perform_rowset_gc.simulation.slow", { auto target_tablet_id = dp->param("tablet_id", -1); @@ -895,9 +855,7 @@ void DataDir::_perform_rowset_gc(const std::string& tablet_schema_hash_path) { return !rowsets_in_version_map.contains(rowset_id) && !_engine.check_rowset_id_in_unused_rowsets(rowset_id) && RowsetMetaManager::exists(get_meta(), tablet->tablet_uid(), rowset_id) - .is() && - !RowsetMetaManager::row_binlog_meta_exists(get_meta(), tablet->tablet_uid(), - rowset_id); + .is(); }; // rowset_id -> is_garbage diff --git a/be/src/storage/iterator/vcollect_iterator.cpp b/be/src/storage/iterator/vcollect_iterator.cpp index 8b2a786e1a6c09..12839bde1a29c5 100644 --- a/be/src/storage/iterator/vcollect_iterator.cpp +++ b/be/src/storage/iterator/vcollect_iterator.cpp @@ -76,7 +76,7 @@ void VCollectIterator::init(TabletReader* reader, bool ori_data_overlapping, boo _merge = false; } - if (_reader->_reader_type == ReaderType::READER_BINLOG) { + if (_reader->_reader_context.read_row_binlog) { _merge = false; } diff --git a/be/src/storage/iterator/vertical_merge_iterator.cpp b/be/src/storage/iterator/vertical_merge_iterator.cpp index 114d535b917c3b..a5bf00d02c7426 100644 --- a/be/src/storage/iterator/vertical_merge_iterator.cpp +++ b/be/src/storage/iterator/vertical_merge_iterator.cpp @@ -204,8 +204,6 @@ Status RowSourcesBuffer::_create_buffer_file() { file_path_ss << "_full"; } else if (_reader_type == ReaderType::READER_COLD_DATA_COMPACTION) { file_path_ss << "_cold"; - } else if (_reader_type == ReaderType::READER_BINLOG_COMPACTION) { - file_path_ss << "_binlog"; } else { DCHECK(false); return Status::InternalError("unknown reader type"); diff --git a/be/src/storage/iterators.h b/be/src/storage/iterators.h index c8d5a313bab5a6..28e8fd96222fef 100644 --- a/be/src/storage/iterators.h +++ b/be/src/storage/iterators.h @@ -129,6 +129,7 @@ class StorageReadOptions { // For rows with the same key, use ascending order (small-to-large) for tie-breakers. // For example, use lower rowset version / segment id first. bool use_insert_order_when_same = false; + bool read_row_binlog = false; int binlog_tso_idx = -1; // columns for orderby keys std::vector* read_orderby_key_columns = nullptr; diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index 1cebd5ef4ab093..5667860149c7dd 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -75,6 +75,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, TabletReader::ReaderParams reader_params; reader_params.tablet = tablet; reader_params.reader_type = reader_type; + reader_params.read_row_binlog = tablet->is_row_binlog_tablet(); TabletReadSource read_source; read_source.rs_splits.reserve(src_rowset_readers.size()); @@ -97,8 +98,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } - if (reader_params.reader_type == ReaderType::READER_BINLOG_COMPACTION) { - reader_params.delete_bitmap = tablet->tablet_meta()->binlog_delvec_ptr(); + if (reader_params.read_row_binlog) { + reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } if (stats_output && stats_output->rowid_conversion) { @@ -261,6 +262,7 @@ Status Merger::vertical_compact_one_group( reader_params.key_group_cluster_key_idxes = key_group_cluster_key_idxes; reader_params.tablet = tablet; reader_params.reader_type = reader_type; + reader_params.read_row_binlog = tablet->is_row_binlog_tablet(); reader_params.enable_sparse_optimization = enable_sparse_optimization; TabletReadSource read_source; @@ -286,8 +288,8 @@ Status Merger::vertical_compact_one_group( reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); has_cluster_key = true; } - if (reader_params.reader_type == ReaderType::READER_BINLOG_COMPACTION) { - reader_params.delete_bitmap = tablet->tablet_meta()->binlog_delvec_ptr(); + if (reader_params.read_row_binlog) { + reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } if (is_key && stats_output && stats_output->rowid_conversion) { diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index c13963b3bff449..12ca1d2e6a458f 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -60,7 +60,8 @@ enum CompactionType { BASE_COMPACTION = 1, CUMULATIVE_COMPACTION = 2, FULL_COMPACTION = 3, - BINLOG_COMPACTION = 4 + // Only used by scheduler to route row-binlog tablets to the binlog thread pool. + CUMU_BINLOG_COMPACTION = 4 }; enum DataDirType { diff --git a/be/src/storage/olap_server.cpp b/be/src/storage/olap_server.cpp index f562f3d96f4915..844c6c779d4bac 100644 --- a/be/src/storage/olap_server.cpp +++ b/be/src/storage/olap_server.cpp @@ -61,6 +61,7 @@ #include "storage/compaction/cold_data_compaction.h" #include "storage/compaction/compaction_permit_limiter.h" #include "storage/compaction/cumulative_compaction.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction_task_tracker.h" @@ -181,7 +182,7 @@ CompactionSubmitRegistry::TabletSet& CompactionSubmitRegistry::_get_tablet_set( return _tablet_submitted_cumu_compaction[dir]; case CompactionType::FULL_COMPACTION: return _tablet_submitted_full_compaction[dir]; - case CompactionType::BINLOG_COMPACTION: + case CompactionType::CUMU_BINLOG_COMPACTION: return _tablet_submitted_binlog_compaction[dir]; default: CHECK(false) << "invalid compaction type"; @@ -784,13 +785,12 @@ void StorageEngine::_binlog_compaction_tasks_producer_callback() { } std::vector tablet_compaction_contexts = - _generate_compaction_tasks(CompactionType::BINLOG_COMPACTION, data_dirs, + _generate_compaction_tasks(CompactionType::CUMU_BINLOG_COMPACTION, data_dirs, check_score); for (const auto& tablet_compaction_context : tablet_compaction_contexts) { const auto& tablet = tablet_compaction_context.tablet; - Status st = - _submit_compaction_task(tablet, CompactionType::BINLOG_COMPACTION, false, 0, - tablet_compaction_context.prefer_compaction_level); + Status st = _submit_compaction_task(tablet, CompactionType::CUMU_BINLOG_COMPACTION, + false, 0); if (!st.ok()) { LOG(WARNING) << "failed to submit binlog compaction task for tablet: " << tablet->tablet_id() << ", err: " << st; @@ -821,7 +821,7 @@ void StorageEngine::get_tablet_rowset_versions(const PGetTabletVersionsRequest* bool need_generate_compaction_tasks(int task_cnt_per_disk, int thread_per_disk, CompactionType compaction_type, bool all_base) { - if (compaction_type == CompactionType::BINLOG_COMPACTION) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { return task_cnt_per_disk < thread_per_disk; } @@ -869,7 +869,7 @@ int get_concurrent_per_disk(int max_score, int thread_per_disk) { } int32_t disk_compaction_slot_num(const DataDir& data_dir, CompactionType compaction_type) { - if (compaction_type == CompactionType::BINLOG_COMPACTION) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { return config::binlog_compaction_task_num_per_disk; } return data_dir.is_ssd_disk() ? config::compaction_task_num_per_fast_disk @@ -903,9 +903,9 @@ std::vector StorageEngine::_generate_compaction_tasks( for (auto* data_dir : data_dirs) { bool need_pick_tablet = true; uint32_t executing_task_num = - compaction_type == CompactionType::BINLOG_COMPACTION + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION ? compaction_registry_snapshot.count_executing_compaction( - data_dir, CompactionType::BINLOG_COMPACTION) + data_dir, CompactionType::CUMU_BINLOG_COMPACTION) : compaction_registry_snapshot.count_executing_cumu_and_base(data_dir); need_pick_tablet = has_free_compaction_slot(&compaction_registry_snapshot, data_dir, compaction_type, executing_task_num); @@ -944,7 +944,7 @@ std::vector StorageEngine::_generate_compaction_tasks( } else if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value( max_compaction_score); - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { DorisMetrics::instance()->tablet_binlog_max_compaction_score->set_value( max_compaction_score); } @@ -961,6 +961,9 @@ void StorageEngine::_update_cumulative_compaction_policy() { _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( CUMULATIVE_TIME_SERIES_POLICY); + _cumulative_compaction_policies[CUMULATIVE_BINLOG_POLICY] = + CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy( + CUMULATIVE_BINLOG_POLICY); } } @@ -986,7 +989,7 @@ void StorageEngine::_pop_tablet_from_submitted_compaction(TabletSharedPtr tablet Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, CompactionType compaction_type, bool force, - int trigger_method, int8_t prefer_compaction_level) { + int trigger_method) { bool already_exist = _compaction_submit_registry.insert(tablet, compaction_type); if (already_exist) { return Status::AlreadyExist( @@ -996,12 +999,12 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, tablet->compaction_stage = CompactionStage::PENDING; std::shared_ptr compaction; int64_t permits = 0; - Status st = Tablet::prepare_compaction_and_calculate_permits( - compaction_type, tablet, compaction, permits, prefer_compaction_level); + Status st = Tablet::prepare_compaction_and_calculate_permits(compaction_type, tablet, + compaction, permits); if (st.ok() && permits > 0) { if (!force) { - if (compaction_type == CompactionType::BINLOG_COMPACTION) { - if (!_permit_limiter.try_request(permits, compaction_type)) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + if (!_permit_limiter.try_request(permits)) { _pop_tablet_from_submitted_compaction(tablet, compaction_type); tablet->compaction_stage = CompactionStage::NOT_SCHEDULED; return Status::OK(); @@ -1029,8 +1032,8 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, case CompactionType::FULL_COMPACTION: info.compaction_type = CompactionProfileType::FULL; break; - case CompactionType::BINLOG_COMPACTION: - info.compaction_type = CompactionProfileType::BINLOG; + case CompactionType::CUMU_BINLOG_COMPACTION: + info.compaction_type = CompactionProfileType::CUMULATIVE; break; default: DCHECK(false) << "invalid compaction type: " << compaction_type; @@ -1060,7 +1063,7 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, thread_pool = &_cumu_compaction_thread_pool; compaction_type_name = "CUMU"; break; - case CompactionType::BINLOG_COMPACTION: + case CompactionType::CUMU_BINLOG_COMPACTION: thread_pool = &_binlog_compaction_thread_pool; compaction_type_name = "BINLOG"; break; @@ -1096,7 +1099,7 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, // Cleanup tracker on submit failure tracker->remove_task(compaction_id); if (!force) { - _permit_limiter.release(permits, compaction_type); + _permit_limiter.release(permits); } _pop_tablet_from_submitted_compaction(tablet, compaction_type); tablet->compaction_stage = CompactionStage::NOT_SCHEDULED; @@ -1133,7 +1136,7 @@ void StorageEngine::_handle_compaction(TabletSharedPtr tablet, DorisMetrics::instance()->base_compaction_task_running_total->increment(1); DorisMetrics::instance()->base_compaction_task_pending_total->set_value( _base_compaction_thread_pool->get_queue_size()); - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { DorisMetrics::instance()->binlog_compaction_task_running_total->increment(1); DorisMetrics::instance()->binlog_compaction_task_pending_total->set_value( _binlog_compaction_thread_pool->get_queue_size()); @@ -1144,7 +1147,7 @@ void StorageEngine::_handle_compaction(TabletSharedPtr tablet, // Idempotent cleanup: remove task from tracker CompactionTaskTracker::instance()->remove_task(compaction_id); if (!force) { - _permit_limiter.release(permits, compaction_type); + _permit_limiter.release(permits); } _pop_tablet_from_submitted_compaction(tablet, compaction_type); tablet->compaction_stage = CompactionStage::NOT_SCHEDULED; @@ -1161,7 +1164,7 @@ void StorageEngine::_handle_compaction(TabletSharedPtr tablet, DorisMetrics::instance()->base_compaction_task_running_total->increment(-1); DorisMetrics::instance()->base_compaction_task_pending_total->set_value( _base_compaction_thread_pool->get_queue_size()); - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { DorisMetrics::instance()->binlog_compaction_task_running_total->increment(-1); DorisMetrics::instance()->binlog_compaction_task_pending_total->set_value( _binlog_compaction_thread_pool->get_queue_size()); @@ -1262,16 +1265,7 @@ Status StorageEngine::submit_compaction_task(TabletSharedPtr tablet, CompactionT _get_cumulative_compaction_policy(tablet->tablet_meta()->compaction_policy())); } tablet->set_skip_compaction(false); - int8_t prefer_compaction_level = -1; - if (compaction_type == CompactionType::BINLOG_COMPACTION) { - tablet->calc_compaction_score(compaction_type, &prefer_compaction_level); - if (prefer_compaction_level < 0) { - return Status::Error( - "failed to init binlog compaction due to no suitable version"); - } - } - return _submit_compaction_task(tablet, compaction_type, force, trigger_method, - prefer_compaction_level); + return _submit_compaction_task(tablet, compaction_type, force, trigger_method); } Status StorageEngine::_handle_seg_compaction(std::shared_ptr worker, diff --git a/be/src/storage/rowset/beta_rowset_reader.cpp b/be/src/storage/rowset/beta_rowset_reader.cpp index 708346e8414997..30ec41f3861a35 100644 --- a/be/src/storage/rowset/beta_rowset_reader.cpp +++ b/be/src/storage/rowset/beta_rowset_reader.cpp @@ -219,6 +219,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context _read_options.topn_filter_target_node_id = _read_context->topn_filter_target_node_id; _read_options.read_orderby_key_reverse = _read_context->read_orderby_key_reverse; _read_options.use_insert_order_when_same = _read_context->use_insert_order_when_same; + _read_options.read_row_binlog = _read_context->read_row_binlog; int32_t tso_col_id = _read_context->tablet_schema->binlog_tso_col_idx(); if (tso_col_id >= 0) { for (size_t i = 0; i < _read_context->return_columns->size(); ++i) { diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 07680ca99a9324..3fa097554f5dae 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -1104,7 +1104,8 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch rowset_meta->set_data_disk_size(total_data_size + _total_data_size); rowset_meta->set_index_disk_size(total_index_size + _total_index_size); bool aggregate_key_bounds = config::enable_aggregate_non_mow_key_bounds && - !_context.enable_unique_key_merge_on_write; + !_context.enable_unique_key_merge_on_write && + !_rowset_meta->is_row_binlog(); rowset_meta->set_segments_key_bounds(segments_encoded_key_bounds, aggregate_key_bounds); // TODO write zonemap to meta rowset_meta->set_empty((num_rows_written + _num_rows_written) == 0); diff --git a/be/src/storage/rowset/rowset.cpp b/be/src/storage/rowset/rowset.cpp index f1d0ce0747dcd2..fbdefc49233dce 100644 --- a/be/src/storage/rowset/rowset.cpp +++ b/be/src/storage/rowset/rowset.cpp @@ -108,10 +108,10 @@ bool Rowset::check_rowset_segment() { std::string Rowset::get_rowset_info_str() { std::string disk_size = PrettyPrinter::print( static_cast(_rowset_meta->total_disk_size()), TUnit::BYTES); - return fmt::format("[{}-{}] {} {} {} {} {}", start_version(), end_version(), num_segments(), - _rowset_meta->has_delete_predicate() ? "DELETE" : "DATA", + return fmt::format("[{}-{}] {} {} {} {} level={} {}", start_version(), end_version(), + num_segments(), _rowset_meta->has_delete_predicate() ? "DELETE" : "DATA", SegmentsOverlapPB_Name(_rowset_meta->segments_overlap()), - rowset_id().to_string(), disk_size); + _rowset_meta->compaction_level(), rowset_id().to_string(), disk_size); } const TabletSchemaSPtr& Rowset::tablet_schema() const { diff --git a/be/src/storage/rowset/rowset_fwd.h b/be/src/storage/rowset/rowset_fwd.h index 2e741f4e1f5fae..09778025e961ae 100644 --- a/be/src/storage/rowset/rowset_fwd.h +++ b/be/src/storage/rowset/rowset_fwd.h @@ -18,13 +18,18 @@ #pragma once #include +#include namespace doris { +struct Version; +struct HashOfVersion; + class Rowset; using RowsetSharedPtr = std::shared_ptr; class RowsetMeta; using RowsetMetaSharedPtr = std::shared_ptr; +using RowsetMetaMapContainer = std::unordered_map; class RowsetReader; using RowsetReaderSharedPtr = std::shared_ptr; class RowsetWriter; diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index 0903fe127effb1..fde75545091d6e 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -34,7 +34,6 @@ #include "io/fs/encrypted_fs_factory.h" #include "io/fs/file_system.h" #include "runtime/memory/lru_cache_policy.h" -#include "storage/compaction/binlog_compaction_policy.h" #include "storage/metadata_adder.h" #include "storage/olap_common.h" #include "storage/rowset/rowset_fwd.h" @@ -321,14 +320,6 @@ class RowsetMeta : public MetadataAdder { // if segments are overlapping, the score equals to the number of segments, // otherwise, score is 1. uint32_t get_compaction_score() const { - // Row binlog LMax Base([0-x]) only performs meta-only merge, so treat it as score 1. - if (is_row_binlog() && - _rowset_meta_pb.compaction_level() == - BinlogCompactionPolicy::kBinlogCompactionMaxLevel - 1 && - start_version() == 0) { - return 1; - } - uint32_t score = 0; if (!is_segments_overlapping()) { score = 1; @@ -542,8 +533,6 @@ class RowsetMeta : public MetadataAdder { std::atomic _stale_at_s {0}; }; -using RowsetMetaMapContainer = std::unordered_map; - } // namespace doris #endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H diff --git a/be/src/storage/rowset/rowset_meta_manager.cpp b/be/src/storage/rowset/rowset_meta_manager.cpp index 2b9e48d093808b..d442c56328dac0 100644 --- a/be/src/storage/rowset/rowset_meta_manager.cpp +++ b/be/src/storage/rowset/rowset_meta_manager.cpp @@ -74,7 +74,7 @@ Status RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, Status RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, const RowsetMetaPB& rowset_meta_pb, std::optional binlog_format, - const std::map* attach_rowset_map) { + const std::optional& attach_row_binlog_rowset_meta) { if (rowset_meta_pb.partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << rowset_meta_pb.partition_id() << " tablet " << rowset_meta_pb.tablet_id(); @@ -96,9 +96,9 @@ Status RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const Rowse return _save_with_ccr_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb); } DCHECK_EQ(*binlog_format, BinlogFormatPB::ROW); - DCHECK(attach_rowset_map != nullptr); - DCHECK(!attach_rowset_map->empty()); - return _save_with_row_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb, *attach_rowset_map); + DCHECK(attach_row_binlog_rowset_meta.has_value()); + return _save_with_row_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb, + *attach_row_binlog_rowset_meta); } Status RowsetMetaManager::_save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, @@ -160,37 +160,34 @@ Status RowsetMetaManager::_save_with_ccr_binlog(OlapMeta* meta, TabletUid tablet return meta->put(META_COLUMN_FAMILY_INDEX, entries); } -Status RowsetMetaManager::_save_with_row_binlog( - OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, - const RowsetMetaPB& rowset_meta_pb, - const std::map& attach_rowset_map) { - std::vector keys; - std::vector values; - keys.reserve(attach_rowset_map.size() + 1); - values.reserve(attach_rowset_map.size() + 1); - - keys.emplace_back( - fmt::format("{}{}_{}", ROWSET_PREFIX, tablet_uid.to_string(), rowset_id.to_string())); - if (!rowset_meta_pb.SerializeToString(&values.emplace_back())) { +Status RowsetMetaManager::_save_with_row_binlog(OlapMeta* meta, TabletUid tablet_uid, + const RowsetId& rowset_id, + const RowsetMetaPB& rowset_meta_pb, + const RowsetMetaPB& attach_row_binlog_rowset_meta) { + std::string rowset_key = + fmt::format("{}{}_{}", ROWSET_PREFIX, tablet_uid.to_string(), rowset_id.to_string()); + std::string rowset_value; + if (!rowset_meta_pb.SerializeToString(&rowset_value)) { return Status::Error("serialize rowset pb failed. rowset id:{}", - keys[0]); + rowset_key); } - for (const auto& [row_binlog_rs_id, row_binlog_rs_meta_pb] : attach_rowset_map) { - keys.emplace_back(make_row_binlog_key(tablet_uid, rowset_id, row_binlog_rs_id)); - DCHECK(row_binlog_rs_meta_pb.has_is_row_binlog() && row_binlog_rs_meta_pb.is_row_binlog()) - << row_binlog_rs_meta_pb.ShortDebugString(); - if (!row_binlog_rs_meta_pb.SerializeToString(&values.emplace_back())) { - return Status::Error( - "serialize rowset pb failed. rowset id:{}", keys.back()); - } + // save binlog rowset meta as a normal rowset under its own tablet uid in the same batch. + TabletUid row_binlog_tablet_uid(attach_row_binlog_rowset_meta.tablet_uid()); + RowsetId row_binlog_rowset_id; + row_binlog_rowset_id.init(attach_row_binlog_rowset_meta.rowset_id_v2()); + std::string row_binlog_rowset_key = + fmt::format("{}{}_{}", ROWSET_PREFIX, row_binlog_tablet_uid.to_string(), + row_binlog_rowset_id.to_string()); + std::string row_binlog_rowset_value; + if (!attach_row_binlog_rowset_meta.SerializeToString(&row_binlog_rowset_value)) { + return Status::Error("serialize rowset pb failed. rowset id:{}", + row_binlog_rowset_key); } - std::vector entries; - entries.reserve(keys.size()); - for (size_t i = 0; i < keys.size(); ++i) { - entries.emplace_back(keys[i], values[i]); - } + std::vector entries = { + {std::cref(rowset_key), std::cref(rowset_value)}, + {std::cref(row_binlog_rowset_key), std::cref(row_binlog_rowset_value)}}; return meta->put(META_COLUMN_FAMILY_INDEX, entries); } @@ -519,78 +516,6 @@ Status RowsetMetaManager::remove_binlog(OlapMeta* meta, const std::string& suffi kBinlogDataPrefix.data() + suffix}); } -Status RowsetMetaManager::remove_row_binlog(OlapMeta* meta, TabletUid tablet_uid, - const RowsetId& base_rowset_id, - const RowsetId& row_binlog_rowset_id) { - return meta->remove(META_COLUMN_FAMILY_INDEX, - make_row_binlog_key(tablet_uid, base_rowset_id, row_binlog_rowset_id)); -} - -Status RowsetMetaManager::remove_row_binlog_metas(OlapMeta* meta, TabletUid tablet_uid, - const std::set& row_binlog_rowset_ids) { - std::map base_rowset_id_to_row_binlog; - RETURN_IF_ERROR(get_row_binlog_base_rowset_ids(meta, tablet_uid, base_rowset_id_to_row_binlog, - row_binlog_rowset_ids)); - for (const auto& [base_rowset_id, row_binlog_rowset_id] : base_rowset_id_to_row_binlog) { - RETURN_IF_ERROR(remove_row_binlog(meta, tablet_uid, base_rowset_id, row_binlog_rowset_id)); - } - return Status::OK(); -} - -bool RowsetMetaManager::row_binlog_meta_exists(OlapMeta* meta, TabletUid tablet_uid, - const RowsetId& row_binlog_rowset_id) { - bool found = false; - auto probe = [&found, &row_binlog_rowset_id](std::string_view key, - std::string_view /* value */) -> bool { - std::vector parts; - // key format: binlog_row_uuid_{rowset_id}_{row_binlog_rowset_id} - RETURN_IF_ERROR(split_string(key, '_', &parts)); - if (parts.size() != 5) { - LOG(WARNING) << "invalid binlog key:" << key << ", splitted size:" << parts.size(); - return true; - } - RowsetId id; - id.init(parts[4]); - if (id == row_binlog_rowset_id) { - found = true; - return false; - } - return true; - }; - static_cast(meta->iterate(META_COLUMN_FAMILY_INDEX, - std::string(kRowBinlogPrefix) + tablet_uid.to_string(), probe)); - return found; -} - -Status RowsetMetaManager::get_row_binlog_base_rowset_ids( - OlapMeta* meta, TabletUid tablet_uid, - std::map& base_rowset_id_to_row_binlog, - const std::set& row_binlog_rowset_ids) { - auto collect_row_binlog_base_rowset_id = - [&base_rowset_id_to_row_binlog, &row_binlog_rowset_ids]( - std::string_view key, std::string_view /* value */) -> bool { - std::vector parts; - // key format: binlog_row_uuid_{rowset_id}_{row_binlog_rowset_id} - RETURN_IF_ERROR(split_string(key, '_', &parts)); - if (parts.size() != 5) { - LOG(WARNING) << "invalid binlog key:" << key << ", splitted size:" << parts.size(); - return true; - } - - RowsetId rowset_id; - rowset_id.init(parts[3]); - RowsetId row_binlog_rowset_id; - row_binlog_rowset_id.init(parts[4]); - if (row_binlog_rowset_ids.contains(row_binlog_rowset_id)) { - base_rowset_id_to_row_binlog.emplace(rowset_id, row_binlog_rowset_id); - } - return true; - }; - return meta->iterate(META_COLUMN_FAMILY_INDEX, - std::string(kRowBinlogPrefix) + tablet_uid.to_string(), - collect_row_binlog_base_rowset_id); -} - Status RowsetMetaManager::ingest_binlog_metas(OlapMeta* meta, TabletUid tablet_uid, RowsetBinlogMetasPB* metas_pb) { std::vector entries; @@ -672,36 +597,6 @@ Status RowsetMetaManager::traverse_binlog_metas( return status; } -Status RowsetMetaManager::traverse_row_binlog_metas( - OlapMeta* meta, std::function const& func) { - auto traverse_row_binlog_rowset_meta_func = [&func](std::string_view key, - std::string_view value) -> bool { - std::vector parts; - // key format: binlog_row_uuid_{rowset_id}_{row_binlog_rowset_id} - RETURN_IF_ERROR(split_string(key, '_', &parts)); - if (parts.size() != 5) { - LOG(WARNING) << "invalid rowset key:" << key << ", splitted size:" << parts.size(); - return true; - } - std::vector uid_parts; - RETURN_IF_ERROR(split_string(parts[2], '-', &uid_parts)); - if (uid_parts.size() != 2) { - LOG(WARNING) << "invalid tablet uid in binlog key:" << key - << ", splitted size:" << uid_parts.size(); - return true; - } - TabletUid tablet_uid(uid_parts[0], uid_parts[1]); - RowsetId rowset_id; - rowset_id.init(parts[3]); - RowsetId row_binlog_rowset_id; - row_binlog_rowset_id.init(parts[4]); - return func(tablet_uid, rowset_id, row_binlog_rowset_id, std::string(value)); - }; - return meta->iterate(META_COLUMN_FAMILY_INDEX, std::string(kRowBinlogPrefix), - traverse_row_binlog_rowset_meta_func); -} - Status RowsetMetaManager::save_partial_update_info( OlapMeta* meta, int64_t tablet_id, int64_t partition_id, int64_t txn_id, const PartialUpdateInfoPB& partial_update_info_pb) { diff --git a/be/src/storage/rowset/rowset_meta_manager.h b/be/src/storage/rowset/rowset_meta_manager.h index 2007f43829a0da..6afdd80aa3c970 100644 --- a/be/src/storage/rowset/rowset_meta_manager.h +++ b/be/src/storage/rowset/rowset_meta_manager.h @@ -57,10 +57,11 @@ class RowsetMetaManager { static Status get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta); // TODO(Drogon): refactor save && _save_with_binlog to one, adapt to ut temperately - static Status save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, - const RowsetMetaPB& rowset_meta_pb, - std::optional binlog_format = std::nullopt, - const std::map* attach_rowset_map = nullptr); + static Status save( + OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, + const RowsetMetaPB& rowset_meta_pb, + std::optional binlog_format = std::nullopt, + const std::optional& attach_row_binlog_rowset_meta = std::nullopt); // STATEMENT_AND_SNAPSHOT static Status save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, @@ -89,20 +90,6 @@ class RowsetMetaManager { static Status ingest_binlog_metas(OlapMeta* meta, TabletUid tablet_uid, RowsetBinlogMetasPB* metas_pb); - static Status remove_row_binlog(OlapMeta* meta, TabletUid tablet_uid, - const RowsetId& base_rowset_id, - const RowsetId& row_binlog_rowset_id); - static Status remove_row_binlog_metas(OlapMeta* meta, TabletUid tablet_uid, - const std::set& row_binlog_rowset_ids); - static bool row_binlog_meta_exists(OlapMeta* meta, TabletUid tablet_uid, - const RowsetId& row_binlog_rowset_id); - static Status get_row_binlog_base_rowset_ids( - OlapMeta* meta, TabletUid tablet_uid, - std::map& base_rowset_id_to_row_binlog, - const std::set& row_binlog_rowset_ids); - static Status traverse_row_binlog_metas( - OlapMeta* meta, std::function const& func); static Status traverse_rowset_metas(OlapMeta* meta, std::function const& collector); @@ -136,7 +123,7 @@ class RowsetMetaManager { static Status _save_with_row_binlog(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, const RowsetMetaPB& rowset_meta_pb, - const std::map& attach_rowset_map); + const RowsetMetaPB& attach_row_binlog_rowset_meta); static Status _get_rowset_binlog_metas(OlapMeta* meta, const TabletUid tablet_uid, const std::vector& binlog_versions, RowsetBinlogMetasPB* metas_pb); diff --git a/be/src/storage/rowset/rowset_reader_context.h b/be/src/storage/rowset/rowset_reader_context.h index 7d37b03dc2a037..0c2ab651c16a00 100644 --- a/be/src/storage/rowset/rowset_reader_context.h +++ b/be/src/storage/rowset/rowset_reader_context.h @@ -41,6 +41,7 @@ class TabletSchema; struct RowsetReaderContext { ReaderType reader_type = ReaderType::READER_QUERY; + bool read_row_binlog = false; Version version {-1, -1}; TabletSchemaSPtr tablet_schema = nullptr; std::vector topn_filter_source_node_ids; diff --git a/be/src/storage/rowset_builder.cpp b/be/src/storage/rowset_builder.cpp index bab3016dd02821..5c825c3806c81c 100644 --- a/be/src/storage/rowset_builder.cpp +++ b/be/src/storage/rowset_builder.cpp @@ -138,8 +138,8 @@ void RowsetBuilder::_garbage_collection(bool cancel_txn) { // when rollback failed should not delete rowset if (need_clean) { _engine.add_unused_rowset(_rowset); - for (auto& rs : _attach_rowsets) { - _engine.add_unused_rowset(rs); + if (_attach_row_binlog.rowset != nullptr) { + _engine.add_unused_rowset(_attach_row_binlog.rowset); } } } @@ -395,7 +395,7 @@ Status RowsetBuilder::commit_txn() { // Transfer ownership of `PendingRowsetGuard` to `TxnManager` Status res = _engine.txn_manager()->commit_txn( _req.partition_id, *tablet(), _req.txn_id, _req.load_id, _rowset, - std::move(_pending_rs_guard), false, _partial_update_info, &_attach_rowsets); + std::move(_pending_rs_guard), false, _partial_update_info, _attach_row_binlog); if (!res && !res.is()) { LOG(WARNING) << "Failed to commit txn: " << _req.txn_id @@ -403,7 +403,7 @@ Status RowsetBuilder::commit_txn() { return res; } if (_tablet->enable_unique_key_merge_on_write()) { - // no need to update binlog_delvec, it'll be updated in publish phase + // no need to update binlog tablet delete bitmap, it'll be updated in publish phase _engine.txn_manager()->set_txn_related_delete_bitmap( _req.partition_id, _req.txn_id, tablet()->tablet_id(), tablet()->tablet_uid(), true, _delete_bitmap, *_rowset_ids, _partial_update_info); @@ -430,9 +430,8 @@ Status BaseRowsetBuilder::_build_current_tablet_schema( const TabletSchema& ori_tablet_schema) { const OlapTableIndexSchema* index_schema = nullptr; if (_req.write_req_type == WriteRequestType::ROW_BINLOG) { - const auto* row_binlog_index_schema = table_schema_param->row_binlog_index_schema(); + const auto* row_binlog_index_schema = table_schema_param->row_binlog_index_schema(index_id); DCHECK(row_binlog_index_schema != nullptr); - DCHECK_EQ(row_binlog_index_schema->index_id, index_id); index_schema = row_binlog_index_schema; } else { for (const auto* schema : table_schema_param->indexes()) { @@ -542,6 +541,7 @@ Status GroupRowsetBuilder::init() { cfg.source.mow_context = data_ctx.mow_context; cfg.source.is_transient_rowset_writer = data_ctx.is_transient_rowset_writer; cfg.source.source_write_type = data_ctx.write_type; + cfg.source.base_tablet = _txn_rs_builder->tablet_sptr(); } _rowset_writer = std::move(group_writer); @@ -558,9 +558,13 @@ Status GroupRowsetBuilder::wait_calc_delete_bitmap() { } Status GroupRowsetBuilder::commit_txn() { - // Attach binlog rowset to txn rowset, so that commit/rollback and - // clean-up are all handled by txn rowset builder. - RETURN_IF_ERROR(_txn_rs_builder->attach_rowset_to_txn(_row_binlog_rowset_builder->rowset())); + // Attach binlog rowset (and its independent binlog tablet) to txn rowset, so that + // commit/rollback and clean-up are all handled by txn rowset builder. + RowBinlogTxnInfo attach_row_binlog; + attach_row_binlog.rowset = _row_binlog_rowset_builder->rowset(); + attach_row_binlog.tablet = + std::static_pointer_cast(_row_binlog_rowset_builder->tablet_sptr()); + RETURN_IF_ERROR(_txn_rs_builder->attach_row_binlog_to_txn(attach_row_binlog)); auto st = _txn_rs_builder->commit_txn(); if (st.ok()) { // Avoid RowBinlogRowsetBuilder being cleaned in its base dtor. @@ -575,9 +579,8 @@ Status RowBinlogRowsetBuilder::init() { RETURN_IF_ERROR(_init_context_common_fields(context)); // build tablet schema in request level - RETURN_IF_ERROR(_build_current_tablet_schema( - _req.index_id, _req.table_schema_param.get(), - *std::dynamic_pointer_cast(_tablet)->row_binlog_tablet_schema())); + RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(), + *_tablet->tablet_schema())); context.tablet_schema = _tablet_schema; context.write_binlog_opt().enable = true; @@ -589,11 +592,11 @@ Status RowBinlogRowsetBuilder::init() { return Status::OK(); } -Status BaseRowsetBuilder::attach_rowset_to_txn(const RowsetSharedPtr& rowset) { +Status BaseRowsetBuilder::attach_row_binlog_to_txn(const RowBinlogTxnInfo& attach_row_binlog) { if (!is_data_builder()) { return Status::RuntimeError("the rowset isn't allowed to manage txn"); } - _attach_rowsets.push_back(rowset); + _attach_row_binlog = attach_row_binlog; return Status::OK(); } diff --git a/be/src/storage/rowset_builder.h b/be/src/storage/rowset_builder.h index fe3488c7ff86be..af708be2598b0d 100644 --- a/be/src/storage/rowset_builder.h +++ b/be/src/storage/rowset_builder.h @@ -32,6 +32,7 @@ #include "storage/rowset/pending_rowset_helper.h" #include "storage/rowset/rowset.h" #include "storage/tablet/tablet_fwd.h" +#include "storage/txn/txn_manager.h" namespace doris { @@ -87,8 +88,8 @@ class BaseRowsetBuilder { bool is_data_builder() const { return _req.write_req_type == WriteRequestType::DATA; } - // Attach an extra rowset (e.g. binlog rowset) to the same txn. - Status attach_rowset_to_txn(const RowsetSharedPtr& rowset); + // Attach the binlog rowset and its independent binlog tablet to the same txn. + Status attach_row_binlog_to_txn(const RowBinlogTxnInfo& attach_row_binlog); // Attach an extra pending rowset id so that PendingLocalRowsets can be // cleaned up together with the primary rowset. @@ -113,8 +114,8 @@ class BaseRowsetBuilder { WriteRequest _req; BaseTabletSPtr _tablet; RowsetSharedPtr _rowset; - // Extra rowsets attached to the same txn (e.g. binlog rowsets). - std::vector _attach_rowsets; + // The binlog rowset and its independent binlog tablet attached to the same txn. + RowBinlogTxnInfo _attach_row_binlog; std::shared_ptr _rowset_writer; PendingRowsetGuard _pending_rs_guard; // Extra rowset ids that share the same PendingRowsetGuard. @@ -166,8 +167,8 @@ class RowsetBuilder : public BaseRowsetBuilder { RuntimeProfile::Counter* _commit_txn_timer = nullptr; }; -// Rowset builder dedicated for row_binlog rowset, it shares the same tablet -// but uses an independent row_binlog tablet schema. +// Rowset builder dedicated for row_binlog rowset. It writes to the independent +// row_binlog tablet while being attached to the base tablet txn by GroupRowsetBuilder. class RowBinlogRowsetBuilder : public RowsetBuilder { public: RowBinlogRowsetBuilder(StorageEngine& engine, const WriteRequest& req, RuntimeProfile* profile); @@ -189,9 +190,8 @@ class RowBinlogRowsetBuilder : public RowsetBuilder { } }; -// manage one transaction with multiple rowset_builder -// eg. normal data rowset + row_binlog rowset -// Now only support one tablet +// Manage one transaction with multiple rowset_builders. +// eg. normal data rowset + row_binlog rowset. class GroupRowsetBuilder : public BaseRowsetBuilder { public: GroupRowsetBuilder(StorageEngine& engine, const WriteRequest& group_build_req, diff --git a/be/src/storage/rowset_version_mgr.cpp b/be/src/storage/rowset_version_mgr.cpp index aa6ed733c39658..5040c87cabfe23 100644 --- a/be/src/storage/rowset_version_mgr.cpp +++ b/be/src/storage/rowset_version_mgr.cpp @@ -64,12 +64,10 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch [[nodiscard]] Result> BaseTablet::capture_consistent_versions_unlocked( const Version& version_range, const CaptureRowsetOps& options) const { std::vector version_path; - auto& version_tracker = - options.capture_row_binlog ? _row_binlog_version_tracker : _timestamped_version_tracker; - auto st = version_tracker.capture_consistent_versions(version_range, &version_path); + auto st = + _timestamped_version_tracker.capture_consistent_versions(version_range, &version_path); if (!st && !options.quiet) { - auto missed_versions = - get_missed_versions_unlocked(version_range.second, options.capture_row_binlog); + auto missed_versions = get_missed_versions_unlocked(version_range.second); if (missed_versions.empty()) { LOG(WARNING) << fmt::format( "version already has been merged. version_range={}, max_version={}, " @@ -111,17 +109,14 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch auto rowset_for_version = [&](const Version& version, bool include_stale) -> Result { - const auto& rs_version_map = - options.capture_row_binlog ? _row_binlog_rs_version_map : _rs_version_map; - if (auto it = rs_version_map.find(version); it != rs_version_map.end()) { + if (auto it = _rs_version_map.find(version); it != _rs_version_map.end()) { return it->second; } else { - VLOG_NOTICE << "fail to find Rowset in " - << (options.capture_row_binlog ? "row_binlog_rs_version" : "rs_version") - << " for version. tablet=" << tablet_id() << ", version='" - << version.first << "-" << version.second; + VLOG_NOTICE << "fail to find Rowset in rs_version for version. tablet=" + << tablet_id() << ", version='" << version.first << "-" + << version.second; } - if (!options.capture_row_binlog && include_stale) { + if (include_stale) { if (auto it = _stale_rs_version_map.find(version); it != _stale_rs_version_map.end()) { return it->second; @@ -144,9 +139,7 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch rowsets.push_back(std::move(ret.value())); } - if (options.capture_row_binlog) { - result.delete_bitmap = _tablet_meta->binlog_delvec_ptr(); - } else if (keys_type() == KeysType::UNIQUE_KEYS && enable_unique_key_merge_on_write()) { + if (need_read_delete_bitmap()) { result.delete_bitmap = _tablet_meta->delete_bitmap_ptr(); } return result; @@ -409,9 +402,9 @@ Result BaseTablet::_remote_capture_rowsets( cntl->tablet_id = tablet_id(); cntl->req_addrs = std::move(be_addresses); cntl->version_range = version_range; - bool is_mow = keys_type() == KeysType::UNIQUE_KEYS && enable_unique_key_merge_on_write(); + bool need_delete_bitmap = need_read_delete_bitmap(); CaptureRowsetResult result; - if (is_mow) { + if (need_delete_bitmap) { result.delete_bitmap = std::make_unique(_tablet_meta->delete_bitmap().snapshot()); DeleteBitmapPB delete_bitmap_keys; @@ -446,7 +439,7 @@ Result BaseTablet::_remote_capture_rowsets( } result.rowsets.push_back(std::move(rs)); } - if (is_mow) { + if (need_delete_bitmap) { DCHECK_NE(result.delete_bitmap, nullptr); result.delete_bitmap->merge(*remote_meta.delete_bitmap); } diff --git a/be/src/storage/segment/historical_row_retriever.cpp b/be/src/storage/segment/historical_row_retriever.cpp index b56daef47cb31a..963b83d0c489f6 100644 --- a/be/src/storage/segment/historical_row_retriever.cpp +++ b/be/src/storage/segment/historical_row_retriever.cpp @@ -42,7 +42,6 @@ #include "storage/rowset/rowset_writer_context.h" #include "storage/segment/segment.h" #include "storage/storage_engine.h" -#include "storage/tablet/tablet.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" @@ -83,7 +82,7 @@ Status PrimaryKeyModelRowRetriever::init(const HistoricalRowRetrieverContext& co Status PrimaryKeyModelRowRetriever::retrieve_historical_row(const Int8* delete_sign_column_data, size_t row_pos, size_t num_rows) { - auto* tablet = static_cast(_context.tablet.get()); + auto* tablet = _context.tablet.get(); auto& tablet_schema = _context.tablet_schema; DCHECK(_context.partial_update_info); @@ -161,9 +160,6 @@ Status PrimaryKeyModelRowRetriever::retrieve_historical_row(const Int8* delete_s Status PrimaryKeyModelRowRetriever::build_after_block(Block* block, size_t row_pos, size_t num_rows) { DCHECK_EQ(_use_default_or_null_flag.size(), num_rows); - if (config::is_cloud_mode()) { - return Status::NotSupported("fill_missing_columns"); - } if (_context.partial_update_info == nullptr) { return Status::InternalError("partial update info is null"); } @@ -175,11 +171,6 @@ Status PrimaryKeyModelRowRetriever::build_after_block(Block* block, size_t row_p Status PrimaryKeyModelRowRetriever::build_before_block(Block* before_block, const std::vector& value_cids, size_t /*row_pos*/, size_t num_rows) { - if (config::is_cloud_mode()) { - // TODO(plat1ko): cloud mode - return Status::NotSupported("fill_before_columns"); - } - auto& tablet_schema = _context.tablet_schema; if (num_rows == 0 || value_cids.empty()) { diff --git a/be/src/storage/segment/row_binlog_segment_writer.cpp b/be/src/storage/segment/row_binlog_segment_writer.cpp index 6a5737a613fad2..e92be5042e6991 100644 --- a/be/src/storage/segment/row_binlog_segment_writer.cpp +++ b/be/src/storage/segment/row_binlog_segment_writer.cpp @@ -20,7 +20,6 @@ #include #include -#include "cloud/config.h" #include "common/cast_set.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" @@ -88,28 +87,25 @@ Status RowBinlogSegmentWriter::init() { _write_before = true; } + auto base_tablet = + _binlog_opts.source.base_tablet == nullptr ? _tablet : _binlog_opts.source.base_tablet; HistoricalRowRetrieverContext historical_row_retriever_context = { - .tablet = _tablet, + .tablet = base_tablet, .tablet_schema = source_schema, .rowset_writer_ctx = _opts.rowset_ctx, .partial_update_info = _binlog_opts.source.partial_update_info, .is_transient_rowset_writer = _binlog_opts.source.is_transient_rowset_writer, .write_type = _binlog_opts.source.source_write_type}; - if (_tablet->enable_unique_key_merge_on_write()) { + if (base_tablet->enable_unique_key_merge_on_write()) { _historical_data_writer = std::make_unique(); RETURN_IF_ERROR(_historical_data_writer->init(historical_row_retriever_context)); - } else if (_tablet->keys_type() == KeysType::AGG_KEYS) { + } else if (base_tablet->keys_type() == KeysType::AGG_KEYS) { // todo } return Status::OK(); } Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, size_t num_rows) { - if (config::is_cloud_mode()) { - // TODO(cjh): cloud mode - return Status::NotSupported("append binlog"); - } - if (_opts.write_type != DataWriteType::TYPE_DIRECT) { // append block directly because binlog data is completed RETURN_IF_ERROR(_append_direct_block(block, row_pos, num_rows)); diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 9f41d84bc29510..da4fcd82d414ad 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -371,8 +371,7 @@ bool Segment::is_tso_placeholder_col(int cid, const Schema& schema, if (read_options.version.first != read_options.version.second) { return false; } - if (read_options.io_ctx.reader_type != ReaderType::READER_BINLOG && - read_options.io_ctx.reader_type != ReaderType::READER_BINLOG_COMPACTION) { + if (!read_options.read_row_binlog) { return false; } // tso_col_idx() is -1 for non-binlog schemas, so this returns false there. diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index bc04ffa2aca276..d830d3099bd0db 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -2460,8 +2460,7 @@ void SegmentIterator::_update_tso_col_if_needed(const std::vector& col return; } - if (_opts.io_ctx.reader_type != ReaderType::READER_BINLOG && - _opts.io_ctx.reader_type != ReaderType::READER_BINLOG_COMPACTION) { + if (!_opts.read_row_binlog) { return; } diff --git a/be/src/storage/snapshot/snapshot_manager.cpp b/be/src/storage/snapshot/snapshot_manager.cpp index 70c789dec82307..7ccacb21d506ef 100644 --- a/be/src/storage/snapshot/snapshot_manager.cpp +++ b/be/src/storage/snapshot/snapshot_manager.cpp @@ -202,7 +202,6 @@ Result> SnapshotManager::convert_rowset_ids( // keep this just for safety new_tablet_meta_pb.clear_inc_rs_metas(); new_tablet_meta_pb.clear_stale_rs_metas(); - new_tablet_meta_pb.clear_row_binlog_rs_metas(); // should modify tablet id and schema hash because in restore process the tablet id is not // equal to tablet id in meta @@ -218,14 +217,11 @@ Result> SnapshotManager::convert_rowset_ids( new_tablet_meta_pb.set_schema_hash(schema_hash); TabletSchemaSPtr tablet_schema = std::make_shared(); tablet_schema->init_from_pb(new_tablet_meta_pb.schema()); - TabletSchemaSPtr row_binlog_tablet_schema = std::make_shared(); - row_binlog_tablet_schema->init_from_pb(new_tablet_meta_pb.row_binlog_schema()); std::unordered_map rs_version_map; std::unordered_map rowset_id_mapping; guards.reserve(cloned_tablet_meta_pb.rs_metas_size() + - cloned_tablet_meta_pb.stale_rs_metas_size() + - cloned_tablet_meta_pb.row_binlog_rs_metas_size()); + cloned_tablet_meta_pb.stale_rs_metas_size()); for (auto&& visible_rowset : cloned_tablet_meta_pb.rs_metas()) { RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_rs_metas(); if (!visible_rowset.has_resource_id()) { @@ -303,35 +299,6 @@ Result> SnapshotManager::convert_rowset_ids( } } - std::string row_binlog_dir = fmt::format("{}/{}", clone_dir, FDRowBinlogSuffix); - for (auto&& row_binlog_rowset : cloned_tablet_meta_pb.row_binlog_rs_metas()) { - RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_row_binlog_rs_metas(); - - if (!row_binlog_rowset.has_resource_id()) { - RowsetId rowset_id = _engine.next_rowset_id(); - guards.push_back(_engine.pending_local_rowsets().add(rowset_id)); - RETURN_IF_ERROR_RESULT(_rename_rowset_id( - row_binlog_rowset, row_binlog_dir, row_binlog_tablet_schema, rowset_id, - rowset_meta, new_tablet_meta_pb.enable_unique_key_merge_on_write())); - RowsetId src_rs_id; - if (row_binlog_rowset.rowset_id() > 0) { - src_rs_id.init(row_binlog_rowset.rowset_id()); - } else { - src_rs_id.init(row_binlog_rowset.rowset_id_v2()); - } - rowset_id_mapping[src_rs_id] = rowset_id; - rowset_meta->set_source_rowset_id(row_binlog_rowset.rowset_id_v2()); - rowset_meta->set_source_tablet_id(cloned_tablet_meta_pb.tablet_id()); - } else { - *rowset_meta = row_binlog_rowset; - } - - rowset_meta->set_tablet_id(tablet_id); - if (partition_id != -1) { - rowset_meta->set_partition_id(partition_id); - } - } - if (new_tablet_meta_pb.schema().index_size() > 0) { RETURN_IF_ERROR_RESULT( _rename_index_ids(*new_tablet_meta_pb.mutable_schema(), target_tablet_schema)); @@ -469,10 +436,6 @@ std::string SnapshotManager::_get_header_full_path(const TabletSharedPtr& ref_ta return fmt::format("{}/{}.hdr", schema_hash_path, ref_tablet->tablet_id()); } -std::string SnapshotManager::_get_row_binlog_full_path(const std::string& schema_hash_path) const { - return fmt::format("{}/{}", schema_hash_path, FDRowBinlogSuffix); -} - std::string SnapshotManager::_get_json_header_full_path(const TabletSharedPtr& ref_tablet, const std::string& schema_hash_path) const { return fmt::format("{}/{}.hdr.json", schema_hash_path, ref_tablet->tablet_id()); @@ -516,19 +479,11 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet return res; } - bool is_copy_binlog = request.__isset.is_copy_binlog ? request.is_copy_binlog : false; - int32_t copy_type = is_copy_binlog ? TabletCopyType::DEFAULT : TTabletCopyType::DATA; - if (request.__isset.copy_type) { - copy_type = request.copy_type; - } - RETURN_IF_ERROR(TabletCopyType::validate(copy_type)); - bool need_row_binlog = TabletCopyType::has(copy_type, TTabletCopyType::ROW_BINLOG); - bool need_ccr_binlog = TabletCopyType::has(copy_type, TTabletCopyType::CCR_BINLOG); + bool need_ccr_binlog = request.__isset.is_copy_binlog ? request.is_copy_binlog : false; // schema_full_path_desc.filepath: // /snapshot_id_path/tablet_id/schema_hash/ auto schema_full_path = get_schema_hash_full_path(target_tablet, snapshot_id_path); - auto row_binlog_full_path = _get_row_binlog_full_path(schema_full_path); // header_path: // /schema_full_path/tablet_id.hdr auto header_path = _get_header_full_path(target_tablet, schema_full_path); @@ -546,7 +501,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize(snapshot_id_path, &snapshot_id)); std::vector consistent_rowsets; - std::vector row_binlog_rowsets; do { TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta()); if (new_tablet_meta == nullptr) { @@ -554,7 +508,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet break; } DeleteBitmap delete_bitmap_snapshot(new_tablet_meta->tablet_id()); - DeleteBitmap binlog_delvec_snapshot(new_tablet_meta->tablet_id()); /// If set missing_version, try to get all missing version. /// If some of them not exist in tablet, we will fall back to @@ -564,12 +517,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (ref_tablet->tablet_state() == TABLET_SHUTDOWN) { return Status::Aborted("tablet has shutdown"); } - if (need_row_binlog && !ref_tablet->enable_row_binlog()) { - res = Status::InternalError( - "tablet {} does not enable row binlog, but ROW_BINLOG copy is requested", - ref_tablet->tablet_id()); - break; - } bool is_single_rowset_clone = (request.__isset.start_version && request.__isset.end_version); if (is_single_rowset_clone) { @@ -588,23 +535,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet "failed to find version when do compaction snapshot"); break; } - if (need_row_binlog) { - RowsetSharedPtr row_binlog_rowset = - ref_tablet->get_row_binlog_rowset_by_version(version); - if (row_binlog_rowset != nullptr) { - row_binlog_rowsets.push_back(row_binlog_rowset); - } else { - LOG(WARNING) - << "failed to find row binlog when do compaction snapshot. " - << " tablet=" << request.tablet_id - << " schema_hash=" << request.schema_hash << " version=" << version; - res = Status::InternalError( - "failed to find row binlog when do compaction snapshot. tablet={}, " - "schema_hash={}, version={}", - request.tablet_id, request.schema_hash, version.to_string()); - break; - } - } } // be would definitely set it as true no matter has missed version or not // but it would take no effects on the following range loop @@ -630,19 +560,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet request.tablet_id, request.schema_hash, version.to_string()); break; } - if (need_row_binlog) { - RowsetSharedPtr row_binlog_rowset = - ref_tablet->get_row_binlog_rowset_by_version(version); - if (row_binlog_rowset != nullptr) { - row_binlog_rowsets.push_back(row_binlog_rowset); - } else { - res = Status::InternalError( - "failed to find row binlog when snapshot. tablet={}, " - "schema_hash={}, version={}", - request.tablet_id, request.schema_hash, version.to_string()); - break; - } - } } } @@ -670,7 +587,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet /// not all missing versions are found, fall back to full snapshot. res = Status::OK(); // reset res consistent_rowsets.clear(); // reset vector - row_binlog_rowsets.clear(); // get latest version const RowsetSharedPtr last_version = ref_tablet->get_rowset_with_max_version(); @@ -728,20 +644,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet LOG(WARNING) << "fail to select versions to span. res=" << res; break; } - if (need_row_binlog) { - // row binlog does not support cooldown yet. - auto ret = ref_tablet->capture_consistent_rowsets_unlocked( - Version(0, version), CaptureRowsetOps {.capture_row_binlog = true}); - if (ret) { - row_binlog_rowsets = std::move(ret->rowsets); - } else { - res = std::move(ret.error()); - } - if (!res.ok()) { - LOG(WARNING) << "fail to select row binlogs to span. res=" << res; - break; - } - } *allow_incremental_clone = false; } else { version = ref_tablet->max_version_unlocked(); @@ -758,10 +660,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet ref_tablet->enable_unique_key_merge_on_write()) { delete_bitmap_snapshot = ref_tablet->tablet_meta()->delete_bitmap().snapshot(version); - if (need_row_binlog) { - binlog_delvec_snapshot = - ref_tablet->tablet_meta()->binlog_delvec().snapshot(version); - } } } @@ -786,48 +684,13 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet << " ref tablet=" << ref_tablet->tablet_id(); break; } - - std::vector row_binlog_rs_metas; - if (need_row_binlog) { - res = io::global_local_filesystem()->create_directory(row_binlog_full_path); - if (!res.ok()) { - break; - } - for (auto& rs : row_binlog_rowsets) { - if (rs->is_local()) { - // local row binlog - res = rs->link_files_to(row_binlog_full_path, rs->rowset_id()); - if (!res.ok()) { - break; - } - } - row_binlog_rs_metas.push_back(rs->rowset_meta()); - VLOG_NOTICE << "add row binlog rowset meta to clone list. " - << " start version " << rs->rowset_meta()->start_version() - << " end version " << rs->rowset_meta()->end_version() << " empty " - << rs->rowset_meta()->empty(); - } - if (!res.ok()) { - LOG(WARNING) << "fail to create row binlog hard link. path=" << snapshot_id_path - << " tablet=" << target_tablet->tablet_id() - << " ref tablet=" << ref_tablet->tablet_id(); - break; - } - } - // The inc_rs_metas is deprecated since Doris version 0.13. // Clear it for safety reason. // Whether it is incremental or full snapshot, rowset information is stored in rs_meta. new_tablet_meta->revise_rs_metas(std::move(rs_metas)); - if (need_row_binlog) { - new_tablet_meta->revise_row_binlog_rs_metas(std::move(row_binlog_rs_metas)); - } if (ref_tablet->keys_type() == UNIQUE_KEYS && ref_tablet->enable_unique_key_merge_on_write()) { new_tablet_meta->revise_delete_bitmap_unlocked(delete_bitmap_snapshot); - if (need_row_binlog) { - new_tablet_meta->revise_binlog_delvec_unlocked(binlog_delvec_snapshot); - } } if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION2) { diff --git a/be/src/storage/snapshot/snapshot_manager.h b/be/src/storage/snapshot/snapshot_manager.h index 37f99d9c55b17b..0db4a586df5db1 100644 --- a/be/src/storage/snapshot/snapshot_manager.h +++ b/be/src/storage/snapshot/snapshot_manager.h @@ -116,8 +116,6 @@ class SnapshotManager { std::string _get_header_full_path(const TabletSharedPtr& ref_tablet, const std::string& schema_hash_path) const; - std::string _get_row_binlog_full_path(const std::string& schema_hash_path) const; - std::string _get_json_header_full_path(const TabletSharedPtr& ref_tablet, const std::string& schema_hash_path) const; diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp index 2bc9e46b101378..064c5d7a4f6229 100644 --- a/be/src/storage/storage_engine.cpp +++ b/be/src/storage/storage_engine.cpp @@ -992,45 +992,6 @@ void StorageEngine::_clean_unused_rowset_metas() { } return true; }; - std::vector> invalid_row_binlog_metas; - auto clean_row_binlog_rowsets = [this, &invalid_row_binlog_metas]( - const TabletUid& tablet_uid, RowsetId rowset_id, - RowsetId row_binlog_rowset_id, - const std::string& meta_str) -> bool { - // return false will break meta iterator, return true to skip this error - RowsetMetaSharedPtr row_binlog_rowset_meta(new RowsetMeta()); - bool parsed = row_binlog_rowset_meta->init(meta_str); - if (!parsed) { - LOG(WARNING) << "parse binlog meta string failed for rowset_id:" - << row_binlog_rowset_id; - row_binlog_rowset_meta->set_rowset_id(row_binlog_rowset_id); - row_binlog_rowset_meta->set_tablet_uid(tablet_uid); - invalid_row_binlog_metas.emplace_back(rowset_id, row_binlog_rowset_meta); - return true; - } - TabletSharedPtr tablet = _tablet_manager->get_tablet(row_binlog_rowset_meta->tablet_id()); - if (tablet == nullptr) { - LOG(INFO) << "failed to find tablet " << row_binlog_rowset_meta->tablet_id() - << " for binlog: " << row_binlog_rowset_meta->rowset_id() - << ", tablet may be dropped"; - invalid_row_binlog_metas.emplace_back(rowset_id, row_binlog_rowset_meta); - return true; - } - if (tablet->tablet_uid() != row_binlog_rowset_meta->tablet_uid()) { - LOG(WARNING) << "binlog meta's tablet uid " << row_binlog_rowset_meta->tablet_uid() - << " does not equal to tablet uid: " << tablet->tablet_uid(); - invalid_row_binlog_metas.emplace_back(rowset_id, row_binlog_rowset_meta); - return true; - } - if (row_binlog_rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && - !tablet->rowset_meta_is_useful(row_binlog_rowset_meta) && - !check_rowset_id_in_unused_rowsets(rowset_id)) { - LOG(INFO) << "binlog meta is not used any more, remove it. rowset_id=" - << row_binlog_rowset_meta->rowset_id(); - invalid_row_binlog_metas.emplace_back(rowset_id, row_binlog_rowset_meta); - } - return true; - }; auto data_dirs = get_stores(); for (auto data_dir : data_dirs) { static_cast( @@ -1060,16 +1021,6 @@ void StorageEngine::_clean_unused_rowset_metas() { LOG(INFO) << "remove " << invalid_rowset_metas.size() << " invalid rowset meta from dir: " << data_dir->path(); - static_cast(RowsetMetaManager::traverse_row_binlog_metas(data_dir->get_meta(), - clean_row_binlog_rowsets)); - for (auto& rs_id_to_meta : invalid_row_binlog_metas) { - static_cast(RowsetMetaManager::remove_row_binlog( - data_dir->get_meta(), rs_id_to_meta.second->tablet_uid(), rs_id_to_meta.first, - rs_id_to_meta.second->rowset_id())); - } - LOG(INFO) << "remove " << invalid_row_binlog_metas.size() - << " invalid binlog meta from dir: " << data_dir->path(); - invalid_row_binlog_metas.clear(); invalid_rowset_metas.clear(); } } diff --git a/be/src/storage/storage_engine.h b/be/src/storage/storage_engine.h index ddf48002f39fb3..5530b09dae29e0 100644 --- a/be/src/storage/storage_engine.h +++ b/be/src/storage/storage_engine.h @@ -201,6 +201,7 @@ class BaseStorageEngine { std::unique_ptr _base_compaction_thread_pool; std::unique_ptr _cumu_compaction_thread_pool; + std::unique_ptr _binlog_compaction_thread_pool; int _cumu_compaction_thread_pool_used_threads {0}; int _cumu_compaction_thread_pool_small_tasks_running {0}; }; @@ -448,8 +449,7 @@ class StorageEngine final : public BaseStorageEngine { CompactionType compaction_type); Status _submit_compaction_task(TabletSharedPtr tablet, CompactionType compaction_type, - bool force, int trigger_method = 0, - int8_t prefer_compaction_level = -1); + bool force, int trigger_method = 0); void _handle_compaction(TabletSharedPtr tablet, std::shared_ptr compaction, CompactionType compaction_type, int64_t permits, bool force, @@ -540,8 +540,6 @@ class StorageEngine final : public BaseStorageEngine { // Type of new loaded data RowsetTypePB _default_rowset_type; - std::unique_ptr _binlog_compaction_thread_pool; - std::unique_ptr _seg_compaction_thread_pool; std::unique_ptr _cold_data_compaction_thread_pool; diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index 13b8b7baaec408..97629ac82d0f75 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -135,8 +135,6 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) : _tablet_meta(std::move // construct _timestamped_versioned_tracker from rs and stale rs meta _timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas(), _tablet_meta->all_stale_rs_metas()); - _row_binlog_version_tracker.construct_versioned_tracker( - _tablet_meta->all_row_binlog_rs_metas()); // if !_tablet_meta->all_rs_metas()[0]->tablet_schema(), // that mean the tablet_meta is still no upgrade to doris 1.2 versions. @@ -256,14 +254,6 @@ RowsetSharedPtr BaseTablet::get_stale_rowset_by_version(const Version& version) return iter->second; } -RowsetSharedPtr BaseTablet::get_row_binlog_rowset_by_version(const Version& version) const { - auto iter = _row_binlog_rs_version_map.find(version); - if (iter == _row_binlog_rs_version_map.end()) { - return nullptr; - } - return iter->second; -} - // Already under _meta_lock RowsetSharedPtr BaseTablet::get_rowset_with_max_version() const { Version max_version = _tablet_meta->max_version(); @@ -323,14 +313,11 @@ Versions BaseTablet::get_missed_versions(int64_t spec_version) const { return calc_missed_versions(spec_version, std::move(existing_versions)); } -Versions BaseTablet::get_missed_versions_unlocked(int64_t spec_version, - bool capture_row_binlog) const { +Versions BaseTablet::get_missed_versions_unlocked(int64_t spec_version) const { DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version; Versions existing_versions; - const auto& rs_metas = capture_row_binlog ? _tablet_meta->all_row_binlog_rs_metas() - : _tablet_meta->all_rs_metas(); - for (const auto& [ver, _] : rs_metas) { + for (const auto& [ver, _] : _tablet_meta->all_rs_metas()) { existing_versions.emplace_back(ver); } return calc_missed_versions(spec_version, std::move(existing_versions)); @@ -348,15 +335,10 @@ void BaseTablet::_print_missed_versions(const Versions& missed_versions) const { bool BaseTablet::_reconstruct_version_tracker_if_necessary() { double data_orphan_vertex_ratio = _timestamped_version_tracker.get_orphan_vertex_ratio(); - double row_binlog_orphan_vertex_ratio = _row_binlog_version_tracker.get_orphan_vertex_ratio(); if (data_orphan_vertex_ratio >= config::tablet_version_graph_orphan_vertex_ratio) { _timestamped_version_tracker.construct_versioned_tracker( _tablet_meta->all_rs_metas(), _tablet_meta->all_stale_rs_metas()); return true; - } else if (row_binlog_orphan_vertex_ratio >= config::tablet_version_graph_orphan_vertex_ratio) { - _row_binlog_version_tracker.construct_versioned_tracker( - _tablet_meta->all_row_binlog_rs_metas()); - return true; } return false; } @@ -1525,13 +1507,14 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf DeleteBitmapPtr delete_bitmap = txn_info->delete_bitmap; bool is_partial_update = txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update(); - for (const auto& rs : txn_info->attach_rowsets) { - if (rs != nullptr && rs->rowset_meta() != nullptr && rs->rowset_meta()->is_row_binlog()) { - row_binlog_rowset = rs; - build_row_binlog = is_partial_update || - self->tablet_meta()->binlog_config().need_historical_value(); - break; - } + const auto& binlog_rs = txn_info->attach_row_binlog.rowset; + if (binlog_rs != nullptr && binlog_rs->rowset_meta() != nullptr && + binlog_rs->rowset_meta()->is_row_binlog()) { + DCHECK(txn_info->attach_row_binlog.tablet != nullptr); + row_binlog_rowset = binlog_rs; + build_row_binlog = is_partial_update || txn_info->attach_row_binlog.tablet->tablet_meta() + ->binlog_config() + .need_historical_value(); } // rewrite conflict only when partial update or need before @@ -1662,8 +1645,9 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf DCHECK(transient_rs_writer != nullptr); // Create transient row binlog writer for publish-phase segment appending. - auto transient_row_binlog_writer = DORIS_TRY(self->create_transient_rowset_writer( - *row_binlog_rowset, txn_info->partial_update_info, txn_expiration)); + auto transient_row_binlog_writer = + DORIS_TRY(txn_info->attach_row_binlog.tablet->create_transient_rowset_writer( + *row_binlog_rowset, txn_info->partial_update_info, txn_expiration)); // Prepare source MOW context for historical row retrieval in binlog writer. auto& data_ctx = const_cast(transient_rs_writer->context()); @@ -1679,6 +1663,7 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf cfg.source.is_transient_rowset_writer = data_ctx.is_transient_rowset_writer; cfg.source.source_write_type = data_ctx.write_type; cfg.source.row_binlog_rowset = row_binlog_rowset; + cfg.source.base_tablet = self; // Wrap two transient writers into a group writer for dual flush/build. RowsetWriterSharedPtr data_writer_sp(std::move(transient_rs_writer)); diff --git a/be/src/storage/tablet/base_tablet.h b/be/src/storage/tablet/base_tablet.h index 68980e4d9d1f6c..290722a05d453f 100644 --- a/be/src/storage/tablet/base_tablet.h +++ b/be/src/storage/tablet/base_tablet.h @@ -89,6 +89,13 @@ class BaseTablet : public std::enable_shared_from_this { return _tablet_meta->enable_unique_key_merge_on_write(); } + bool need_read_delete_bitmap() const { + return _tablet_meta->enable_unique_key_merge_on_write() || + _tablet_meta->is_row_binlog_tablet(); + } + + bool is_row_binlog_tablet() const { return _tablet_meta->is_row_binlog_tablet(); } + // Property encapsulated in TabletMeta const TabletMetaSharedPtr& tablet_meta() const { return _tablet_meta; } @@ -104,11 +111,6 @@ class BaseTablet : public std::enable_shared_from_this { return _max_version_schema; } - TabletSchemaSPtr row_binlog_tablet_schema() const { - std::shared_lock rlock(_meta_lock); - return _tablet_meta->row_binlog_schema(); - } - void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; } bool is_alter_failed() { return _alter_failed; } @@ -141,7 +143,6 @@ class BaseTablet : public std::enable_shared_from_this { // The caller must call hold _meta_lock when call this three function. RowsetSharedPtr get_rowset_by_version(const Version& version, bool find_is_stale = false) const; RowsetSharedPtr get_stale_rowset_by_version(const Version& version) const; - RowsetSharedPtr get_row_binlog_rowset_by_version(const Version& version) const; RowsetSharedPtr get_rowset_with_max_version() const; Status get_all_rs_id(int64_t max_version, RowsetIdUnorderedSet* rowset_ids) const; @@ -149,8 +150,7 @@ class BaseTablet : public std::enable_shared_from_this { // Get the missed versions until the spec_version. Versions get_missed_versions(int64_t spec_version) const; - Versions get_missed_versions_unlocked(int64_t spec_version, - bool capture_row_binlog = false) const; + Versions get_missed_versions_unlocked(int64_t spec_version) const; void generate_tablet_meta_copy(TabletMeta& new_tablet_meta, bool cloud_get_rowset_meta) const; void generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta, @@ -382,7 +382,6 @@ class BaseTablet : public std::enable_shared_from_this { mutable BthreadSharedMutex _meta_lock; TimestampedVersionTracker _timestamped_version_tracker; - TimestampedVersionTracker _row_binlog_version_tracker; // After version 0.13, all newly created rowsets are saved in _rs_version_map. // And if rowset being compacted, the old rowsets will be saved in _stale_rs_version_map; @@ -391,8 +390,6 @@ class BaseTablet : public std::enable_shared_from_this { // These _stale rowsets are been removed when rowsets' pathVersion is expired, // this policy is judged and computed by TimestampedVersionTracker. std::unordered_map _stale_rs_version_map; - // for row_binlog - std::unordered_map _row_binlog_rs_version_map; const TabletMetaSharedPtr _tablet_meta; TabletSchemaSPtr _max_version_schema; @@ -466,7 +463,6 @@ struct CaptureRowsetOps { bool quiet = false; bool include_stale_rowsets = true; bool enable_fetch_rowsets_from_peers = false; - bool capture_row_binlog = false; // ======== only take effect in cloud mode ======== diff --git a/be/src/storage/tablet/tablet.cpp b/be/src/storage/tablet/tablet.cpp index 25189c329213ab..455b11ebea425c 100644 --- a/be/src/storage/tablet/tablet.cpp +++ b/be/src/storage/tablet/tablet.cpp @@ -81,9 +81,8 @@ #include "service/point_query_executor.h" #include "storage/binlog.h" #include "storage/compaction/base_compaction.h" -#include "storage/compaction/binlog_compaction.h" -#include "storage/compaction/binlog_compaction_policy.h" #include "storage/compaction/cumulative_compaction.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction/full_compaction.h" @@ -162,9 +161,6 @@ void set_last_failure_time(Tablet* tablet, const Compaction& compaction, int64_t case ReaderType::READER_FULL_COMPACTION: tablet->set_last_full_compaction_failure_time(ms); return; - case ReaderType::READER_BINLOG_COMPACTION: - tablet->set_last_binlog_compaction_failure_time(ms); - return; default: LOG(FATAL) << "invalid compaction type " << compaction.compaction_name() << " tablet_id: " << tablet->tablet_id(); @@ -259,7 +255,6 @@ Tablet::Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* _last_cumu_compaction_success_millis(0), _last_base_compaction_success_millis(0), _last_full_compaction_success_millis(0), - _last_binlog_compaction_failure_millis(0), _cumulative_point(K_INVALID_CUMULATIVE_POINT), _newly_created_rowset_num(0), _last_checkpoint_time(0), @@ -271,16 +266,6 @@ Tablet::Tablet(StorageEngine& engine, TabletMetaSharedPtr tablet_meta, DataDir* _tablet_path = fmt::format("{}/{}/{}/{}/{}", _data_dir->path(), DATA_PREFIX, _tablet_meta->shard_id(), tablet_id(), schema_hash()); } - _binlog_compaction_policy = std::make_shared(); - init_last_binlog_compaction_success_and_failure_time(); -} - -void Tablet::init_last_binlog_compaction_success_and_failure_time() { - int64_t now = UnixMillis(); - for (int8_t level = 0; level < BinlogCompactionPolicy::kBinlogCompactionMaxLevel; ++level) { - _last_binlog_compaction_success_millis[level] = now; - } - _last_binlog_compaction_failure_millis = 0; } bool Tablet::set_tablet_schema_into_rowset_meta() { @@ -339,20 +324,6 @@ Status Tablet::_init_once_action() { _stale_rs_version_map[version] = std::move(rowset); } - // init row_binlog rowset - for (const auto& [_, row_binlog_rs_meta] : _tablet_meta->all_row_binlog_rs_metas()) { - Version version = row_binlog_rs_meta->version(); - RowsetSharedPtr rowset; - res = create_rowset(row_binlog_rs_meta, &rowset); - if (!res.ok()) { - LOG(WARNING) << "fail to init binlog rowset. tablet_id:" << tablet_id() - << ", schema_hash:" << schema_hash() << ", version=" << version - << ", res:" << res; - return res; - } - _row_binlog_rs_version_map[version] = std::move(rowset); - } - return res; } @@ -372,20 +343,16 @@ void Tablet::save_meta() { // Caller should hold _meta_lock. Status Tablet::revise_tablet_meta(const std::vector& to_add, const std::vector& to_delete, - bool is_incremental_clone, bool copy_row_binlog) { + bool is_incremental_clone) { LOG(INFO) << "begin to revise tablet. tablet_id=" << tablet_id(); // 1. for incremental clone, we have to add the rowsets first to make it easy to compute // all the delete bitmaps, and it's easy to delete them if we end up with a failure // 2. for full clone, we can calculate delete bitmaps on the cloned rowsets directly. if (is_incremental_clone) { CHECK(to_delete.empty()); // don't need to delete rowsets - add_rowsets(to_add, copy_row_binlog); + add_rowsets(to_add); // reconstruct from tablet meta _timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas()); - if (copy_row_binlog) { - _row_binlog_version_tracker.construct_versioned_tracker( - _tablet_meta->all_row_binlog_rs_metas()); - } } Status calc_bm_status; @@ -394,9 +361,6 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, normal_rowsets.reserve(to_add.size()); base_rowsets_for_full_clone.reserve(to_add.size()); for (const auto& rs : to_add) { - if (copy_row_binlog && rs->rowset_meta()->is_row_binlog()) { - continue; - } normal_rowsets.push_back(rs); base_rowsets_for_full_clone.push_back(rs); } @@ -475,7 +439,7 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, // error handling if (!calc_bm_status.ok()) { if (is_incremental_clone) { - RETURN_IF_ERROR(delete_rowsets(to_add, false, copy_row_binlog)); + RETURN_IF_ERROR(delete_rowsets(to_add, false)); LOG(WARNING) << "incremental clone on tablet: " << tablet_id() << " failed due to " << calc_bm_status.msg() << ", revert " << to_add.size() << " rowsets added before."; @@ -488,14 +452,10 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, // full clone, calculate delete bitmap succeeded, update rowset if (!is_incremental_clone) { - RETURN_IF_ERROR(delete_rowsets(to_delete, false, copy_row_binlog)); - add_rowsets(to_add, copy_row_binlog); + RETURN_IF_ERROR(delete_rowsets(to_delete, false)); + add_rowsets(to_add); // reconstruct from tablet meta _timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas()); - if (copy_row_binlog) { - _row_binlog_version_tracker.construct_versioned_tracker( - _tablet_meta->all_row_binlog_rs_metas()); - } // check the rowsets used for delete bitmap calculation is equal to the rowsets // that we can capture by version @@ -527,15 +487,14 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, return Status::OK(); } -Status Tablet::add_rowset(RowsetSharedPtr rowset, RowsetSharedPtr row_binlog_rowset) { +Status Tablet::add_rowset(RowsetSharedPtr rowset) { DCHECK(rowset != nullptr); std::lock_guard wrlock(_meta_lock); SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD); // If the rowset already exist, just return directly. The rowset_id is an unique-id, // we can use it to check this situation. if (_contains_rowset(rowset->rowset_id())) { - // Ensure binlog is also added on retry. - return _add_row_binlog_rowset_unlocked(rowset, row_binlog_rowset); + return Status::OK(); } // Otherwise, the version should be not contained in any existing rowset. RETURN_IF_ERROR(_contains_version(rowset->version())); @@ -545,8 +504,6 @@ Status Tablet::add_rowset(RowsetSharedPtr rowset, RowsetSharedPtr row_binlog_row _timestamped_version_tracker.add_version(rowset->version()); add_compaction_score(rowset->rowset_meta()->get_compaction_score()); - RETURN_IF_ERROR(_add_row_binlog_rowset_unlocked(rowset, row_binlog_rowset)); - std::vector rowsets_to_delete; // yiguolei: temp code, should remove the rowset contains by this rowset // but it should be removed in multi path version @@ -565,32 +522,6 @@ Status Tablet::add_rowset(RowsetSharedPtr rowset, RowsetSharedPtr row_binlog_row return Status::OK(); } -Status Tablet::_add_row_binlog_rowset_unlocked(const RowsetSharedPtr& rowset, - const RowsetSharedPtr& row_binlog_rowset) { - if (row_binlog_rowset == nullptr) { - return Status::OK(); - } - DCHECK(rowset != nullptr); - DCHECK_EQ(row_binlog_rowset->version(), rowset->version()); - - const auto& version = row_binlog_rowset->version(); - if (auto it = _row_binlog_rs_version_map.find(version); - it != _row_binlog_rs_version_map.end()) { - if (it->second != nullptr && it->second->rowset_id() == row_binlog_rowset->rowset_id()) { - return Status::OK(); - } - return Status::Error( - "binlog version already exists. existing rowset_id={}, version={}, tablet={}", - it->second != nullptr ? it->second->rowset_id().to_string() : "0", - version.to_string(), tablet_id()); - } - - RETURN_IF_ERROR(_tablet_meta->add_row_binlog_rs_meta(row_binlog_rowset->rowset_meta())); - _row_binlog_rs_version_map[version] = row_binlog_rowset; - _row_binlog_version_tracker.add_version(version); - return Status::OK(); -} - bool Tablet::rowset_exists_unlocked(const RowsetSharedPtr& rowset) { if (auto it = _rs_version_map.find(rowset->version()); it == _rs_version_map.end()) { return false; @@ -703,94 +634,29 @@ Status Tablet::modify_rowsets(std::vector& to_add, return Status::OK(); } -Status Tablet::modify_row_binlog_rowsets(std::vector& to_add, - std::vector& to_delete) { - if (to_add.empty() && to_delete.empty()) { - return Status::OK(); - } - - for (auto&& rs : to_delete) { - if (auto it = _row_binlog_rs_version_map.find(rs->version()); - it == _row_binlog_rs_version_map.end()) { - return Status::Error( - "try to delete not exist row-binlog {} from {}", rs->version().to_string(), - tablet_id()); - } else if (rs->rowset_id() != it->second->rowset_id()) { - return Status::Error( - "try to delete row-binlog {} from {}, but rowset id changed, delete rowset id " - "is {}, exists rowsetid is {}", - rs->version().to_string(), tablet_id(), rs->rowset_id().to_string(), - it->second->rowset_id().to_string()); - } - } - - std::vector rs_metas_to_delete; - for (auto& rs : to_delete) { - rs_metas_to_delete.push_back(rs->rowset_meta()); - _row_binlog_rs_version_map.erase(rs->version()); - _row_binlog_version_tracker.delete_version(rs->version()); - // Delete binlog delvec immediately because input rowsets will be deleted. - tablet_meta()->binlog_delvec().remove({rs->rowset_id(), 0, 0}, - {rs->rowset_id(), UINT32_MAX, UINT64_MAX}); - _engine.add_unused_rowset(rs); - } - - std::vector rs_metas_to_add; - for (auto& rs : to_add) { - rs_metas_to_add.push_back(rs->rowset_meta()); - _row_binlog_rs_version_map[rs->version()] = rs; - _row_binlog_version_tracker.add_version(rs->version()); - ++_newly_created_rowset_num; - } - - _tablet_meta->modify_row_binlog_rs_metas(rs_metas_to_add, rs_metas_to_delete); - return Status::OK(); -} - -void Tablet::add_rowsets(const std::vector& to_add, bool copy_row_binlog) { +void Tablet::add_rowsets(const std::vector& to_add) { if (to_add.empty()) { return; } std::vector rs_metas; - std::vector row_binlog_rs_metas; rs_metas.reserve(to_add.size()); - row_binlog_rs_metas.reserve(to_add.size()); for (auto& rs : to_add) { - if (copy_row_binlog && rs->rowset_meta()->is_row_binlog()) { - _row_binlog_rs_version_map.emplace(rs->version(), rs); - _row_binlog_version_tracker.add_version(rs->version()); - row_binlog_rs_metas.push_back(rs->rowset_meta()); - } else { - _rs_version_map.emplace(rs->version(), rs); - _timestamped_version_tracker.add_version(rs->version()); - rs_metas.push_back(rs->rowset_meta()); - } + _rs_version_map.emplace(rs->version(), rs); + _timestamped_version_tracker.add_version(rs->version()); + rs_metas.push_back(rs->rowset_meta()); } _tablet_meta->modify_rs_metas(rs_metas, {}); - if (copy_row_binlog) { - _tablet_meta->modify_row_binlog_rs_metas(row_binlog_rs_metas, {}); - } } -Status Tablet::delete_rowsets(const std::vector& to_delete, bool move_to_stale, - bool copy_row_binlog) { +Status Tablet::delete_rowsets(const std::vector& to_delete, bool move_to_stale) { if (to_delete.empty()) { return Status::OK(); } std::vector rs_metas; - std::vector row_binlog_rs_metas; std::vector normal_rowsets; - std::vector row_binlog_rowsets; rs_metas.reserve(to_delete.size()); - row_binlog_rs_metas.reserve(to_delete.size()); int64_t now = ::time(nullptr); for (const auto& rs : to_delete) { - if (copy_row_binlog && rs->rowset_meta()->is_row_binlog()) { - row_binlog_rs_metas.push_back(rs->rowset_meta()); - _row_binlog_rs_version_map.erase(rs->version()); - row_binlog_rowsets.push_back(rs); - continue; - } if (move_to_stale) { rs->rowset_meta()->set_stale_at(now); } @@ -799,9 +665,6 @@ Status Tablet::delete_rowsets(const std::vector& to_delete, boo normal_rowsets.push_back(rs); } _tablet_meta->modify_rs_metas({}, rs_metas, !move_to_stale); - if (copy_row_binlog) { - _tablet_meta->modify_row_binlog_rs_metas({}, row_binlog_rs_metas); - } if (move_to_stale) { for (const auto& rs : normal_rowsets) { _stale_rs_version_map[rs->version()] = rs; @@ -817,12 +680,6 @@ Status Tablet::delete_rowsets(const std::vector& to_delete, boo } } } - for (const auto& rs : row_binlog_rowsets) { - _row_binlog_version_tracker.delete_version(rs->version()); - if (rs->is_local()) { - _engine.add_unused_rowset(rs); - } - } return Status::OK(); } @@ -842,14 +699,12 @@ RowsetSharedPtr Tablet::_rowset_with_largest_size() { } // add inc rowset should not persist tablet meta, because it will be persisted when publish txn. -Status Tablet::add_inc_rowset(const RowsetSharedPtr& rowset, - const RowsetSharedPtr& row_binlog_rowset) { +Status Tablet::add_inc_rowset(const RowsetSharedPtr& rowset) { DCHECK(rowset != nullptr); std::lock_guard wrlock(_meta_lock); SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD); if (_contains_rowset(rowset->rowset_id())) { - // Ensure binlog is also added on retry. - return _add_row_binlog_rowset_unlocked(rowset, row_binlog_rowset); + return Status::OK(); } RETURN_IF_ERROR(_contains_version(rowset->version())); @@ -862,7 +717,7 @@ Status Tablet::add_inc_rowset(const RowsetSharedPtr& rowset, add_compaction_score(rowset->rowset_meta()->get_compaction_score()); - return _add_row_binlog_rowset_unlocked(rowset, row_binlog_rowset); + return Status::OK(); } void Tablet::_delete_stale_rowset_by_version(const Version& version) { @@ -1154,14 +1009,18 @@ Versions Tablet::calc_missed_versions(int64_t spec_version, Versions existing_ve } bool Tablet::can_do_compaction(size_t path_hash, CompactionType compaction_type) { + if (is_row_binlog_tablet() && compaction_type != CompactionType::CUMU_BINLOG_COMPACTION) { + return false; + } + if (compaction_type == CompactionType::BASE_COMPACTION && tablet_state() != TABLET_RUNNING) { // base compaction can only be done for tablet in TABLET_RUNNING state. // but cumulative compaction can be done for TABLET_NOTREADY, such as tablet under alter process. return false; } - if (compaction_type == CompactionType::BINLOG_COMPACTION && - (!config::enable_feature_binlog || !enable_row_binlog())) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION && + (!config::enable_feature_binlog || !is_row_binlog_tablet())) { return false; } @@ -1174,11 +1033,10 @@ bool Tablet::can_do_compaction(size_t path_hash, CompactionType compaction_type) return tablet_state() == TABLET_RUNNING || tablet_state() == TABLET_NOTREADY; } -uint32_t Tablet::calc_compaction_score(CompactionType compaction_type, - int8_t* prefer_compaction_level) { - if (compaction_type == CompactionType::BINLOG_COMPACTION) { +uint32_t Tablet::calc_compaction_score(CompactionType compaction_type) { + if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { std::shared_lock rdlock(_meta_lock); - return _calc_binlog_compaction_score(prefer_compaction_level); + return _calc_cumulative_compaction_score(_cumulative_compaction_policy); } if (_score_check_cnt++ % config::check_score_rounds_num != 0) { @@ -1210,7 +1068,7 @@ bool Tablet::suitable_for_compaction( std::shared_ptr cumulative_compaction_policy) { #ifndef BE_TEST if ((compaction_type == CompactionType::CUMULATIVE_COMPACTION || - compaction_type == CompactionType::BINLOG_COMPACTION) && + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) && cumulative_compaction_policy != nullptr) { std::lock_guard wrlock(_meta_lock); if (_cumulative_compaction_policy == nullptr || @@ -1223,13 +1081,12 @@ bool Tablet::suitable_for_compaction( // Need meta lock, because it will iterator "all_rs_metas" of tablet meta. std::shared_lock rdlock(_meta_lock); int32_t score = -1; - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION || + compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { score = _calc_cumulative_compaction_score(cumulative_compaction_policy); } else if (compaction_type == CompactionType::BASE_COMPACTION) { DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION); score = _calc_base_compaction_score(); - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { - score = _calc_binlog_compaction_score(); } else { DCHECK(false) << "Unknown compaction type: " << compaction_type; } @@ -1405,6 +1262,10 @@ std::vector Tablet::pick_candidate_rowsets_to_cumulative_compac } std::vector Tablet::pick_candidate_rowsets_to_cumulative_compaction_unlocked() { + if (is_row_binlog_tablet()) { + return pick_candidate_rowsets_to_binlog_compaction(); + } + std::vector candidate_rowsets; if (_cumulative_point == K_INVALID_CUMULATIVE_POINT) { return candidate_rowsets; @@ -1478,8 +1339,8 @@ std::vector Tablet::pick_candidate_rowsets_to_binlog_compaction { std::shared_lock rlock(_meta_lock); bool filter_new_rowset = - _row_binlog_rs_version_map.size() <= config::binlog_compaction_file_count_threshold; - for (const auto& [version, rs] : _row_binlog_rs_version_map) { + _rs_version_map.size() <= config::binlog_compaction_file_count_threshold; + for (const auto& [version, rs] : _rs_version_map) { max_version = std::max(max_version, version.second); if (filter_new_rowset && rs->rowset_meta()->is_singleton_delta() && rs->rowset_meta()->newest_write_timestamp() + @@ -1494,7 +1355,7 @@ std::vector Tablet::pick_candidate_rowsets_to_binlog_compaction max_processable_old_version = std::min(max_processable_old_version, visible_version); } - for (const auto& [version, rs] : _row_binlog_rs_version_map) { + for (const auto& [version, rs] : _rs_version_map) { if (!rs->is_local() || version.first > max_processable_old_version) { continue; } @@ -1505,11 +1366,6 @@ std::vector Tablet::pick_candidate_rowsets_to_binlog_compaction return candidate_rowsets; } -uint32_t Tablet::_calc_binlog_compaction_score(int8_t* prefer_compaction_level) { - DCHECK(_binlog_compaction_policy != nullptr); - return _binlog_compaction_policy->calc_binlog_compaction_score(this, prefer_compaction_level); -} - std::vector Tablet::pick_candidate_rowsets_to_build_inverted_index( const std::set& alter_index_uids, bool is_drop_op) { std::vector candidate_rowsets; @@ -1619,26 +1475,12 @@ void Tablet::get_compaction_status(std::string* json_result) { _last_base_compaction_failure_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last full failure time", _last_full_compaction_failure_millis) - FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last binlog failure time", - _last_binlog_compaction_failure_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last cumulative success time", _last_cumu_compaction_success_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last base success time", _last_base_compaction_success_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last full success time", - _last_full_compaction_success_millis) { - int64_t last_binlog_success_millis = 0; - for (int8_t level = 0; level < BinlogCompactionPolicy::kBinlogCompactionMaxLevel; ++level) { - last_binlog_success_millis = - std::max(last_binlog_success_millis, - _last_binlog_compaction_success_millis[level].load()); - } - rapidjson::Value value; - std::string format_str = ToStringFromUnixMillis(last_binlog_success_millis); - value.SetString(format_str.c_str(), cast_set(format_str.length()), - root.GetAllocator()); - root.AddMember("last binlog success time", value, root.GetAllocator()); - } + _last_full_compaction_success_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last cumulative schedule time", _last_cumu_compaction_schedule_millis) FORMAT_UNIXMILLIS_ADD_JSON_NODE(root, "last base schedule time", @@ -1648,7 +1490,6 @@ void Tablet::get_compaction_status(std::string* json_result) { FORMAT_STRING_ADD_JSON_NODE(root, "last cumulative status", _last_cumu_compaction_status) FORMAT_STRING_ADD_JSON_NODE(root, "last base status", _last_base_compaction_status) FORMAT_STRING_ADD_JSON_NODE(root, "last full status", _last_full_compaction_status) - FORMAT_STRING_ADD_JSON_NODE(root, "last binlog status", _last_binlog_compaction_status) // last single replica compaction status // "single replica compaction status": { @@ -1755,44 +1596,6 @@ bool Tablet::do_tablet_meta_checkpoint() { rs_meta->set_remove_from_rowset_meta(); } - // Remove row binlog metas from rowset meta store after tablet meta is checkpointed. - // Row binlog metas are stored in meta KV with key: - // {kRowBinlogPrefix}{tablet_uid}_{base_rowset_id}_{row_binlog_rowset_id} - // Here we only have row binlog rowset metas, so locate base rowset by version. - const auto& base_rs_metas = _tablet_meta->all_rs_metas(); - const auto& stale_rs_metas = _tablet_meta->all_stale_rs_metas(); - for (const auto& [_, rb_meta] : _tablet_meta->all_row_binlog_rs_metas()) { - // Reuse the same flag to avoid repeated removals across checkpoints. - if (rb_meta->is_remove_from_rowset_meta()) { - continue; - } - - RowsetMetaSharedPtr base_rs_meta; - if (auto base_it = base_rs_metas.find(rb_meta->version()); base_it != base_rs_metas.end()) { - base_rs_meta = base_it->second; - } else if (auto stale_it = stale_rs_metas.find(rb_meta->version()); - stale_it != stale_rs_metas.end()) { - base_rs_meta = stale_it->second; - } - if (base_rs_meta == nullptr) { - LOG(WARNING) << "failed to locate base rowset meta for binlog by version, tablet=" - << tablet_id() << ", version=" << rb_meta->version().to_string() - << ", binlog_rowset_id=" << rb_meta->rowset_id() - << ", try to remove by scanning row-binlog meta store"; - RETURN_FALSE_IF_ERROR(RowsetMetaManager::remove_row_binlog_metas( - _data_dir->get_meta(), tablet_uid(), {rb_meta->rowset_id()})); - rb_meta->set_remove_from_rowset_meta(); - continue; - } - - RETURN_FALSE_IF_ERROR(RowsetMetaManager::remove_row_binlog( - _data_dir->get_meta(), tablet_uid(), base_rs_meta->rowset_id(), - rb_meta->rowset_id())); - VLOG_NOTICE << "remove binlog meta from meta store, base_rowset_id=" - << base_rs_meta->rowset_id() << ", binlog_rowset_id=" << rb_meta->rowset_id(); - rb_meta->set_remove_from_rowset_meta(); - } - if (keys_type() == UNIQUE_KEYS && enable_unique_key_merge_on_write()) { RETURN_FALSE_IF_ERROR(TabletMetaManager::remove_old_version_delete_bitmap( _data_dir, tablet_id(), max_version_unlocked())); @@ -1822,16 +1625,6 @@ bool Tablet::rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta) { find_version = true; } } - if (rowset_meta->is_row_binlog()) { - for (auto& version_rowset : _row_binlog_rs_version_map) { - if (version_rowset.second->rowset_id() == rowset_meta->rowset_id()) { - return true; - } - if (version_rowset.second->contains_version(rowset_meta->version())) { - find_version = true; - } - } - } return !find_version; } @@ -1985,12 +1778,6 @@ void Tablet::build_tablet_report_info(TTabletInfo* tablet_info, tablet_info->__set_local_segment_size(_tablet_meta->tablet_local_segment_size()); tablet_info->__set_remote_index_size(_tablet_meta->tablet_remote_index_size()); tablet_info->__set_remote_segment_size(_tablet_meta->tablet_remote_segment_size()); - if (enable_row_binlog()) { - int64_t total_binlog_size = _tablet_meta->binlog_size(); - int64_t total_binlog_file_num = _tablet_meta->binlog_file_num(); - tablet_info->__set_binlog_file_num(total_binlog_file_num); - tablet_info->__set_binlog_size(total_binlog_size); - } } void Tablet::report_error(const Status& st) { @@ -2008,8 +1795,7 @@ void Tablet::report_error(const Status& st) { Status Tablet::prepare_compaction_and_calculate_permits( CompactionType compaction_type, const TabletSharedPtr& tablet, - std::shared_ptr& compaction, int64_t& permits, - int8_t prefer_compaction_level) { + std::shared_ptr& compaction, int64_t& permits) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { MonotonicStopWatch watch; watch.start(); @@ -2094,24 +1880,19 @@ Status Tablet::prepare_compaction_and_calculate_permits( // And because we set permits to 0, so even if we return OK here, nothing will be done. return Status::OK(); } - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { - MonotonicStopWatch watch; - watch.start(); - - DCHECK_GE(prefer_compaction_level, 0); - compaction = std::make_shared(tablet->_engine, tablet, - prefer_compaction_level); + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION) { + compaction = std::make_shared(tablet->_engine, tablet); Status res = compaction->prepare_compact(); if (!res.ok()) { - tablet->set_last_binlog_compaction_failure_time(UnixMillis()); + tablet->set_last_cumu_compaction_failure_time(UnixMillis()); permits = 0; - if (!res.is()) { + if (!res.is()) { return Status::InternalError("prepare binlog compaction with err: {}", res.to_string()); } - // return OK if OLAP_ERR_BINLOG_COMPACTION_NO_SUITABLE_VERSION, so that we don't need to - // print too much useless logs. + // return OK if there is no suitable version, so that we don't need to + // print too many useless logs. // And because we set permits to 0, so even if we return OK here, nothing will be done. return Status::OK(); } @@ -2198,37 +1979,14 @@ Status Tablet::create_initial_rowset(const int64_t req_version) { return Status::OK(); }; - if (!enable_row_binlog()) { - RowsetWriterContext context; - RowsetSharedPtr new_rowset; - RETURN_IF_ERROR(get_rowset_writer_context(context, tablet_schema())); - auto rs_writer = DORIS_TRY(create_rowset_writer(context, false)); - - RETURN_IF_ERROR(rs_writer->flush()); - RETURN_IF_ERROR(rs_writer->build(new_rowset)); - RETURN_IF_ERROR(add_rowset(std::move(new_rowset), nullptr)); - } else { - std::unique_ptr group_rowset_writer; - RETURN_IF_ERROR(RowsetFactory::create_empty_group_rowset_writer(&group_rowset_writer)); - - RowsetWriterContext data_context; - RETURN_IF_ERROR(get_rowset_writer_context(data_context, tablet_schema())); - auto data_writer = DORIS_TRY(create_rowset_writer(data_context, false)); - group_rowset_writer->set_data_writer(std::move(data_writer)); - - RowsetWriterContext row_binlog_context; - row_binlog_context.write_binlog_opt().enable = true; - RETURN_IF_ERROR(get_rowset_writer_context(row_binlog_context, row_binlog_tablet_schema())); - auto row_binlog_writer = DORIS_TRY(create_rowset_writer(row_binlog_context, false)); - group_rowset_writer->set_row_binlog_writer(std::move(row_binlog_writer)); - - RETURN_IF_ERROR(group_rowset_writer->flush_rowsets()); - - std::vector waited_build_rowsets; + RowsetWriterContext context; + RowsetSharedPtr new_rowset; + RETURN_IF_ERROR(get_rowset_writer_context(context, tablet_schema())); + auto rs_writer = DORIS_TRY(create_rowset_writer(context, false)); - RETURN_IF_ERROR(group_rowset_writer->build_rowsets(waited_build_rowsets)); - RETURN_IF_ERROR(add_rowset(waited_build_rowsets.at(0), waited_build_rowsets.at(1))); - } + RETURN_IF_ERROR(rs_writer->flush()); + RETURN_IF_ERROR(rs_writer->build(new_rowset)); + RETURN_IF_ERROR(add_rowset(std::move(new_rowset))); set_cumulative_layer_point(req_version + 1); return Status::OK(); @@ -2271,11 +2029,9 @@ Result> Tablet::create_transient_rowset_writer( context.write_type = DataWriteType::TYPE_DIRECT; context.partial_update_info = std::move(partial_update_info); context.is_transient_rowset_writer = true; - if (rowset.rowset_meta() != nullptr && rowset.rowset_meta()->is_row_binlog()) { context.write_binlog_opt().enable = true; } - return create_transient_rowset_writer(context, rowset.rowset_id()) .transform([&](auto&& writer) { writer->set_segment_start_id(cast_set(rowset.num_segments())); @@ -2319,10 +2075,8 @@ void Tablet::_init_context_common_fields(RowsetWriterContext& context) { context.encrypt_algorithm = tablet_meta()->encryption_algorithm(); if (context.write_binlog_opt().enable) { - context.tablet_schema_hash = row_binlog_schema_hash(); - bool need_before = tablet_meta()->binlog_config().need_historical_value(); - context.write_binlog_opt().set_need_before(need_before); - context.tablet_path = row_binlog_path(); + context.write_binlog_opt().set_need_before( + tablet_meta()->binlog_config().need_historical_value()); } } @@ -2876,18 +2630,17 @@ Status Tablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, RowsetSharedPtr rowset = txn_info->rowset; int64_t cur_version = rowset->start_version(); - // For binlog publish, sync current rowset delete bitmap deltas to `binlog_delvec` + // For binlog publish, sync current rowset delete bitmap deltas to binlog tablet. // so row binlog reads can skip rows deleted by MOW conflict resolution. - const bool build_row_binlog = !txn_info->attach_rowsets.empty(); + const bool build_row_binlog = txn_info->attach_row_binlog.rowset != nullptr; const RowsetId cur_build_rid = txn_info->rowset->rowset_id(); const RowsetId binlog_rid = - build_row_binlog ? txn_info->attach_rowsets[0]->rowset_id() : cur_build_rid; - auto* binlog_delvec = txn_info->binlog_delvec.get(); + build_row_binlog ? txn_info->attach_row_binlog.rowset->rowset_id() : cur_build_rid; + auto* row_binlog_delete_bitmap = txn_info->attach_row_binlog.delete_bitmap.get(); if (build_row_binlog) { - DCHECK(txn_info->attach_rowsets[0] != nullptr); - DCHECK(txn_info->attach_rowsets[0]->rowset_meta() != nullptr); - DCHECK(txn_info->attach_rowsets[0]->rowset_meta()->is_row_binlog()); - DCHECK(binlog_delvec != nullptr); + DCHECK(txn_info->attach_row_binlog.rowset->rowset_meta() != nullptr); + DCHECK(txn_info->attach_row_binlog.rowset->rowset_meta()->is_row_binlog()); + DCHECK(row_binlog_delete_bitmap != nullptr); } // update version without write lock, compaction and publish_txn @@ -2901,11 +2654,10 @@ Status Tablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, if (build_row_binlog && std::get<0>(key) == cur_build_rid) { const DeleteBitmap::SegmentId& sid = std::get<1>(key); - // Merge current delete bitmap deltas to binlog delvec because publish-phase - // partial-update and seq replace will influence the binlog file. - binlog_delvec->merge({binlog_rid, sid, cast_set(cur_version)}, bitmap); - _tablet_meta->binlog_delvec().merge( - {binlog_rid, sid, cast_set(cur_version)}, bitmap); + // Merge current delete bitmap deltas to binlog tablet because publish-phase + // partial-update and seq replace will influence the binlog rowset. + row_binlog_delete_bitmap->merge({binlog_rid, sid, cast_set(cur_version)}, + bitmap); } } } @@ -2943,7 +2695,7 @@ void Tablet::set_skip_compaction(bool skip, CompactionType compaction_type, int6 _skip_base_compaction = true; _skip_base_compaction_ts = start; } else { - DCHECK(compaction_type == CompactionType::BINLOG_COMPACTION); + DCHECK(compaction_type == CompactionType::CUMU_BINLOG_COMPACTION); _skip_binlog_compaction = true; _skip_binlog_compaction_ts = start; } @@ -2956,7 +2708,8 @@ bool Tablet::should_skip_compaction(CompactionType compaction_type, int64_t now) } else if (compaction_type == CompactionType::BASE_COMPACTION && _skip_base_compaction && now < _skip_base_compaction_ts + config::skip_tablet_compaction_second) { return true; - } else if (compaction_type == CompactionType::BINLOG_COMPACTION && _skip_binlog_compaction && + } else if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION && + _skip_binlog_compaction && now < _skip_binlog_compaction_ts + config::skip_tablet_compaction_second) { return true; } diff --git a/be/src/storage/tablet/tablet.h b/be/src/storage/tablet/tablet.h index 918d892fdeb92c..840ae4599f582a 100644 --- a/be/src/storage/tablet/tablet.h +++ b/be/src/storage/tablet/tablet.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -42,7 +41,7 @@ #include "gen_cpp/AgentService_types.h" #include "storage/binlog.h" #include "storage/binlog_config.h" -#include "storage/compaction/binlog_compaction_policy.h" +#include "storage/compaction/cumulative_compaction_binlog_policy.h" #include "storage/data_dir.h" #include "storage/olap_common.h" #include "storage/partial_update_info.h" @@ -119,14 +118,11 @@ class Tablet final : public BaseTablet { int64_t replica_id() const { return _tablet_meta->replica_id(); } std::string tablet_path() const override { return _tablet_path; } - std::string row_binlog_path() const { - return fmt::format("{}/{}", _tablet_path, FDRowBinlogSuffix); - } std::string get_rowset_path(const RowsetMetaSharedPtr& rowset_meta) const { if (!rowset_meta->is_local()) { return ""; } - return rowset_meta->is_row_binlog() ? row_binlog_path() : tablet_path(); + return tablet_path(); } bool set_tablet_schema_into_rowset_meta(); @@ -142,7 +138,7 @@ class Tablet final : public BaseTablet { // Used in clone task, to update local meta when finishing a clone job Status revise_tablet_meta(const std::vector& to_add, const std::vector& to_delete, - bool is_incremental_clone, bool copy_row_binlog = false); + bool is_incremental_clone); int64_t cumulative_layer_point() const; void set_cumulative_layer_point(int64_t new_point); @@ -163,7 +159,6 @@ class Tablet final : public BaseTablet { uint64_t segment_count() const; Version max_version() const; CumulativeCompactionPolicy* cumulative_compaction_policy(); - BinlogCompactionPolicy* binlog_compaction_policy(); // properties encapsulated in TabletSchema SortType sort_type() const; @@ -177,19 +172,16 @@ class Tablet final : public BaseTablet { int64_t avg_rs_meta_serialize_size() const; // operation in rowsets - Status add_rowset(RowsetSharedPtr rowset, RowsetSharedPtr row_binlog_rowset = nullptr); + Status add_rowset(RowsetSharedPtr rowset); Status create_initial_rowset(const int64_t version); // MUST hold EXCLUSIVE `_meta_lock`. Status modify_rowsets(std::vector& to_add, std::vector& to_delete, bool check_delete = false); - Status modify_row_binlog_rowsets(std::vector& to_add, - std::vector& to_delete); bool rowset_exists_unlocked(const RowsetSharedPtr& rowset); - // Add a committed data rowset and its row binlog rowset - Status add_inc_rowset(const RowsetSharedPtr& rowset, - const RowsetSharedPtr& row_binlog_rowset = nullptr); + // Add a committed rowset. + Status add_inc_rowset(const RowsetSharedPtr& rowset); /// Delete stale rowset by timing. This delete policy uses now() minutes /// config::tablet_rowset_expired_stale_sweep_time_sec to compute the deadline of expired rowset /// to delete. When rowset is deleted, it will be added to StorageEngine unused map and record @@ -219,7 +211,6 @@ class Tablet final : public BaseTablet { std::mutex& get_push_lock() { return _ingest_lock; } std::mutex& get_base_compaction_lock() { return _base_compaction_lock; } std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; } - std::mutex& get_binlog_compaction_lock() { return _binlog_compaction_lock; } std::shared_mutex& get_meta_store_lock() { return _meta_store_lock; } std::shared_timed_mutex& get_migration_lock() { return _migration_lock; } @@ -232,8 +223,7 @@ class Tablet final : public BaseTablet { CompactionType compaction_type, std::shared_ptr cumulative_compaction_policy); - uint32_t calc_compaction_score(CompactionType compaction_type, - int8_t* prefer_compaction_level = nullptr); + uint32_t calc_compaction_score(CompactionType compaction_type); // This function to find max continuous version from the beginning. // For example: If there are 1, 2, 3, 5, 6, 7 versions belongs tablet, then 3 is target. @@ -272,24 +262,6 @@ class Tablet final : public BaseTablet { _last_full_compaction_success_millis = millis; } - int64_t last_binlog_compaction_success_time(int8_t compaction_level) { - DCHECK(compaction_level >= 0 && - compaction_level < BinlogCompactionPolicy::kBinlogCompactionMaxLevel); - return _last_binlog_compaction_success_millis[compaction_level].load(); - } - int64_t last_binlog_compaction_failure_time() { - return _last_binlog_compaction_failure_millis.load(); - } - void init_last_binlog_compaction_success_and_failure_time(); - void set_last_binlog_compaction_success_time(int8_t compaction_level, int64_t millis) { - DCHECK(compaction_level >= 0 && - compaction_level < BinlogCompactionPolicy::kBinlogCompactionMaxLevel); - _last_binlog_compaction_success_millis[compaction_level] = millis; - } - void set_last_binlog_compaction_failure_time(int64_t millis) { - _last_binlog_compaction_failure_millis = millis; - } - int64_t last_cumu_compaction_schedule_time() { return _last_cumu_compaction_schedule_millis; } void set_last_cumu_compaction_schedule_time(int64_t millis) { _last_cumu_compaction_schedule_millis = millis; @@ -346,8 +318,7 @@ class Tablet final : public BaseTablet { static Status prepare_compaction_and_calculate_permits( CompactionType compaction_type, const TabletSharedPtr& tablet, - std::shared_ptr& compaction, int64_t& permits, - int8_t prefer_compaction_level = -1); + std::shared_ptr& compaction, int64_t& permits); void execute_compaction(CompactionMixin& compaction); @@ -378,12 +349,6 @@ class Tablet final : public BaseTablet { std::string get_last_full_compaction_status() { return _last_full_compaction_status; } - void set_last_binlog_compaction_status(std::string status) { - _last_binlog_compaction_status = std::move(status); - } - - std::string get_last_binlog_compaction_status() { return _last_binlog_compaction_status; } - std::tuple get_visible_version_and_time() const; void set_visible_version(const std::shared_ptr& visible_version) { @@ -409,18 +374,14 @@ class Tablet final : public BaseTablet { Status create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset); // MUST hold EXCLUSIVE `_meta_lock` - void add_rowsets(const std::vector& to_add, bool copy_row_binlog = false); + void add_rowsets(const std::vector& to_add); // MUST hold EXCLUSIVE `_meta_lock` - Status delete_rowsets(const std::vector& to_delete, bool move_to_stale, - bool copy_row_binlog = false); + Status delete_rowsets(const std::vector& to_delete, bool move_to_stale); // MUST hold SHARED `_meta_lock` const auto& rowset_map() const { return _rs_version_map; } // MUST hold SHARED `_meta_lock` const auto& stale_rowset_map() const { return _stale_rs_version_map; } - // MUST hold SHARED `_meta_lock` - const auto& row_binlog_rowset_map() const { return _row_binlog_rs_version_map; } - //////////////////////////////////////////////////////////////////////////// // begin cooldown functions //////////////////////////////////////////////////////////////////////////// @@ -520,15 +481,11 @@ class Tablet final : public BaseTablet { return _tablet_meta->binlog_config().is_enable() && _tablet_meta->binlog_config().is_row_binlog_format(); } - int64_t binlog_ttl_ms() const { return _tablet_meta->binlog_config().ttl_seconds(); } int64_t binlog_max_bytes() const { return _tablet_meta->binlog_config().max_bytes(); } void set_binlog_config(BinlogConfig binlog_config); - // row_binlog - int32_t row_binlog_schema_hash() const { return _tablet_meta->row_binlog_schema_hash(); } - void set_is_full_compaction_running(bool is_full_compaction_running) { _is_full_compaction_running = is_full_compaction_running; } @@ -561,9 +518,6 @@ class Tablet final : public BaseTablet { Status _init_once_action(); bool _contains_rowset(const RowsetId rowset_id); Status _contains_version(const Version& version); - Status _add_row_binlog_rowset_unlocked(const RowsetSharedPtr& rowset, - const RowsetSharedPtr& row_binlog_rowset); - // Returns: // version: the max continuous version from beginning // max_version: the max version of this tablet @@ -577,7 +531,6 @@ class Tablet final : public BaseTablet { uint32_t _calc_cumulative_compaction_score( std::shared_ptr cumulative_compaction_policy); uint32_t _calc_base_compaction_score() const; - uint32_t _calc_binlog_compaction_score(int8_t* prefer_compaction_level = nullptr); std::vector _pick_visible_rowsets_to_compaction(int64_t min_start_version, int64_t max_start_version); @@ -622,7 +575,6 @@ class Tablet final : public BaseTablet { std::mutex _ingest_lock; std::mutex _base_compaction_lock; std::mutex _cumulative_compaction_lock; - std::mutex _binlog_compaction_lock; std::shared_timed_mutex _migration_lock; std::mutex _build_inverted_index_lock; @@ -646,11 +598,6 @@ class Tablet final : public BaseTablet { std::atomic _last_base_compaction_success_millis; // timestamp of last full compaction success std::atomic _last_full_compaction_success_millis; - // timestamp of last binlog compaction success for each compaction level - std::array, BinlogCompactionPolicy::kBinlogCompactionMaxLevel> - _last_binlog_compaction_success_millis; - // timestamp of last binlog compaction failure - std::atomic _last_binlog_compaction_failure_millis; // timestamp of last cumu compaction schedule time std::atomic _last_cumu_compaction_schedule_millis; // timestamp of last base compaction schedule time @@ -664,12 +611,10 @@ class Tablet final : public BaseTablet { std::string _last_cumu_compaction_status; std::string _last_base_compaction_status; std::string _last_full_compaction_status; - std::string _last_binlog_compaction_status; // cumulative compaction policy std::shared_ptr _cumulative_compaction_policy; std::string_view _cumulative_compaction_type; - std::shared_ptr _binlog_compaction_policy; // use a separate thread to check all tablets paths existence std::atomic _is_tablet_path_exists; @@ -713,10 +658,6 @@ inline CumulativeCompactionPolicy* Tablet::cumulative_compaction_policy() { return _cumulative_compaction_policy.get(); } -inline BinlogCompactionPolicy* Tablet::binlog_compaction_policy() { - return _binlog_compaction_policy.get(); -} - inline bool Tablet::init_succeeded() { return _init_once.has_called() && _init_once.stored_result().ok(); } @@ -837,26 +778,4 @@ inline int64_t Tablet::avg_rs_meta_serialize_size() const { return _tablet_meta->avg_rs_meta_serialize_size(); } -class TabletCopyType { -public: - static constexpr int32_t DEFAULT = TTabletCopyType::DATA | TTabletCopyType::CCR_BINLOG; - - static bool has(int32_t copy_type, TTabletCopyType::type flag) { - return (copy_type & static_cast(flag)) != 0; - } - - static Status validate(int32_t copy_type) { - if (copy_type <= 0 || (copy_type & ~all_types()) != 0) { - return Status::Error( - "invalid copy_type bitmask: {}, valid bits: {}", copy_type, all_types()); - } - return Status::OK(); - } - -private: - static constexpr int32_t all_types() { - return TTabletCopyType::DATA | TTabletCopyType::ROW_BINLOG | TTabletCopyType::CCR_BINLOG; - } -}; - } // namespace doris diff --git a/be/src/storage/tablet/tablet_fwd.h b/be/src/storage/tablet/tablet_fwd.h index 33dd7824740e60..2c34b004fe11e1 100644 --- a/be/src/storage/tablet/tablet_fwd.h +++ b/be/src/storage/tablet/tablet_fwd.h @@ -38,7 +38,6 @@ using DeleteBitmapPtr = std::shared_ptr; struct TabletCompactionContext { TabletSharedPtr tablet; - int8_t prefer_compaction_level = -1; }; } // namespace doris diff --git a/be/src/storage/tablet/tablet_manager.cpp b/be/src/storage/tablet/tablet_manager.cpp index c27215d3b914b8..cf2a785b87bd3d 100644 --- a/be/src/storage/tablet/tablet_manager.cpp +++ b/be/src/storage/tablet/tablet_manager.cpp @@ -262,11 +262,12 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector // if there have create rollup tablet C(assume on shard-2) from tablet D(assume on shard-1) at the same time, we will meet deadlock std::unique_lock two_tablet_lock(_two_tablet_mtx, std::defer_lock); bool in_restore_mode = request.__isset.in_restore_mode && request.in_restore_mode; - bool is_schema_change_or_atomic_restore = - request.__isset.base_tablet_id && request.base_tablet_id > 0; - bool need_two_lock = - is_schema_change_or_atomic_restore && - ((_tablets_shards_mask & request.base_tablet_id) != (_tablets_shards_mask & tablet_id)); + bool has_base_tablet = request.__isset.base_tablet_id && request.base_tablet_id > 0; + bool is_colocated_row_binlog = + has_base_tablet && request.__isset.is_row_binlog_tablet && request.is_row_binlog_tablet; + bool is_schema_change_or_atomic_restore = has_base_tablet && !is_colocated_row_binlog; + bool need_two_lock = has_base_tablet && ((_tablets_shards_mask & request.base_tablet_id) != + (_tablets_shards_mask & tablet_id)); if (need_two_lock) { SCOPED_TIMER(ADD_TIMER(profile, "GetTwoTableLock")); two_tablet_lock.lock(); @@ -294,8 +295,8 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector } TabletSharedPtr base_tablet = nullptr; - // If the CreateTabletReq has base_tablet_id then it is a alter-tablet request - if (is_schema_change_or_atomic_restore) { + // base_tablet_id is used by alter/restore and colocated row-binlog creation. + if (has_base_tablet) { // if base_tablet_id's lock diffrent with new_tablet_id, we need lock it. if (need_two_lock) { SCOPED_TIMER(ADD_TIMER(profile, "GetBaseTablet")); @@ -317,7 +318,8 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector // entering this method // // ATTN: Since all restored replicas will be saved to HDD, so no storage_medium check here. - if (in_restore_mode || + // Row-binlog tablet must be on the same disk as its base tablet. + if (in_restore_mode || is_colocated_row_binlog || request.storage_medium == base_tablet->data_dir()->storage_medium()) { LOG(INFO) << "create tablet use the base tablet data dir. tablet_id=" << tablet_id << ", base tablet_id=" << request.base_tablet_id @@ -327,9 +329,9 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector } } - // set alter type to schema-change. it is useless TabletSharedPtr tablet = _internal_create_tablet_unlocked( - request, is_schema_change_or_atomic_restore, base_tablet.get(), stores, profile); + request, is_schema_change_or_atomic_restore, is_colocated_row_binlog, base_tablet.get(), + stores, profile); if (tablet == nullptr) { DorisMetrics::instance()->create_tablet_requests_failed->increment(1); return Status::Error("fail to create tablet. tablet_id={}", @@ -342,11 +344,11 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector } TabletSharedPtr TabletManager::_internal_create_tablet_unlocked( - const TCreateTabletReq& request, const bool is_schema_change, const Tablet* base_tablet, + const TCreateTabletReq& request, const bool is_schema_change, + const bool is_colocated_row_binlog, const Tablet* base_tablet, const std::vector& data_dirs, RuntimeProfile* profile) { - // If in schema-change state, base_tablet must also be provided. - // i.e., is_schema_change and base_tablet are either assigned or not assigned - DCHECK((is_schema_change && base_tablet) || (!is_schema_change && !base_tablet)); + DCHECK((is_schema_change && base_tablet) || + (!is_schema_change && (base_tablet == nullptr || is_colocated_row_binlog))); // NOTE: The existence of tablet_id and schema_hash has already been checked, // no need check again here. @@ -480,10 +482,6 @@ TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked( _gen_tablet_dir(data_dir->path(), tablet_meta->shard_id(), request.tablet_id); string schema_hash_dir = path_util::join_path_segments( tablet_dir, std::to_string(request.tablet_schema.schema_hash)); - bool has_row_binlog = tablet_meta->binlog_config().is_enable() && - tablet_meta->binlog_config().is_row_binlog_format(); - string row_binlog_dir = path_util::join_path_segments(schema_hash_dir, "_row_binlog"); - // Because the tablet is removed asynchronously, so that the dir may still exist when BE // receive create-tablet request again, For example retried schema-change request bool exists = true; @@ -501,15 +499,6 @@ TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked( } } - if (has_row_binlog) { - Status st = io::global_local_filesystem()->create_directory(row_binlog_dir); - if (!st.ok()) { - WARN_IF_ERROR(io::global_local_filesystem()->delete_directory(schema_hash_dir), - "failed to cleanup tablet dir after create sub directory failed"); - continue; - } - } - if (tablet_meta->partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet " << tablet_meta->tablet_id(); @@ -737,7 +726,6 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { struct TabletScore { TabletSharedPtr tablet_ptr; uint32_t score = 0; - int8_t prefer_compaction_level = -1; }; std::vector TabletManager::find_best_tablets_to_compaction( @@ -747,7 +735,7 @@ std::vector TabletManager::find_best_tablets_to_compact all_cumulative_compaction_policies) { int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" - : compaction_type == CompactionType::BINLOG_COMPACTION + : compaction_type == CompactionType::CUMU_BINLOG_COMPACTION ? "binlog" : "cumulative"; uint32_t highest_score = 0; @@ -781,8 +769,6 @@ std::vector TabletManager::find_best_tablets_to_compact int64_t last_failure_ms = tablet_ptr->last_cumu_compaction_failure_time(); if (compaction_type == CompactionType::BASE_COMPACTION) { last_failure_ms = tablet_ptr->last_base_compaction_failure_time(); - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { - last_failure_ms = tablet_ptr->last_binlog_compaction_failure_time(); } if (now_ms - last_failure_ms <= config::tablet_sched_delay_time_ms) { VLOG_DEBUG << "Too often to check compaction, skip it. " @@ -799,13 +785,6 @@ std::vector TabletManager::find_best_tablets_to_compact LOG(INFO) << "can not get base lock: " << tablet_ptr->tablet_id(); return; } - } else if (compaction_type == CompactionType::BINLOG_COMPACTION) { - std::unique_lock lock(tablet_ptr->get_binlog_compaction_lock(), - std::try_to_lock); - if (!lock.owns_lock()) { - LOG(INFO) << "can not get binlog lock: " << tablet_ptr->tablet_id(); - return; - } } else { std::unique_lock lock(tablet_ptr->get_cumulative_compaction_lock(), std::try_to_lock); @@ -816,9 +795,7 @@ std::vector TabletManager::find_best_tablets_to_compact } auto cumulative_compaction_policy = all_cumulative_compaction_policies.at( tablet_ptr->tablet_meta()->compaction_policy()); - int8_t prefer_compaction_level = -1; - uint32_t current_compaction_score = - tablet_ptr->calc_compaction_score(compaction_type, &prefer_compaction_level); + uint32_t current_compaction_score = tablet_ptr->calc_compaction_score(compaction_type); if (current_compaction_score < 5) { tablet_ptr->set_skip_compaction(true, compaction_type, UnixSeconds()); } @@ -831,7 +808,6 @@ std::vector TabletManager::find_best_tablets_to_compact TabletScore ts; ts.score = current_compaction_score; ts.tablet_ptr = tablet_ptr; - ts.prefer_compaction_level = prefer_compaction_level; if ((top_tablets.size() >= compaction_num_per_round && current_compaction_score > top_tablets.top().score) || top_tablets.size() < compaction_num_per_round) { @@ -852,8 +828,7 @@ std::vector TabletManager::find_best_tablets_to_compact if (ret) { highest_score = current_compaction_score; best_tablet_context = {.tablet_ptr = tablet_ptr, - .score = current_compaction_score, - .prefer_compaction_level = prefer_compaction_level}; + .score = current_compaction_score}; } } } @@ -866,9 +841,8 @@ std::vector TabletManager::find_best_tablets_to_compact << "compaction_type=" << compaction_type_str << ", tablet_id=" << best_tablet_context.tablet_ptr->tablet_id() << ", path=" << data_dir->path() << ", highest_score=" << highest_score; - picked_tablet_contexts.emplace_back(TabletCompactionContext { - .tablet = std::move(best_tablet_context.tablet_ptr), - .prefer_compaction_level = best_tablet_context.prefer_compaction_level}); + picked_tablet_contexts.emplace_back( + TabletCompactionContext {.tablet = std::move(best_tablet_context.tablet_ptr)}); } std::vector reverse_top_tablets; @@ -878,8 +852,7 @@ std::vector TabletManager::find_best_tablets_to_compact } for (auto it = reverse_top_tablets.rbegin(); it != reverse_top_tablets.rend(); ++it) { - picked_tablet_contexts.emplace_back(TabletCompactionContext { - .tablet = it->tablet_ptr, .prefer_compaction_level = it->prefer_compaction_level}); + picked_tablet_contexts.emplace_back(TabletCompactionContext {.tablet = it->tablet_ptr}); } *score = highest_score; @@ -1120,10 +1093,6 @@ void TabletManager::build_all_report_tablets_info(std::map* t_tablet_stat.__set_local_segment_size(tablet_info.local_segment_size); t_tablet_stat.__set_remote_index_size(tablet_info.remote_index_size); t_tablet_stat.__set_remote_segment_size(tablet_info.remote_segment_size); - if (tablet_info.__isset.binlog_size) { - t_tablet_stat.__set_binlog_size(tablet_info.binlog_size); - t_tablet_stat.__set_binlog_file_num(tablet_info.binlog_file_num); - } }; for_each_tablet(handler, filter_all_tablets); diff --git a/be/src/storage/tablet/tablet_manager.h b/be/src/storage/tablet/tablet_manager.h index 89b7c3fc96e24f..b507e79ada7a8a 100644 --- a/be/src/storage/tablet/tablet_manager.h +++ b/be/src/storage/tablet/tablet_manager.h @@ -206,6 +206,7 @@ class TabletManager { TabletSharedPtr _internal_create_tablet_unlocked(const TCreateTabletReq& request, const bool is_schema_change, + const bool is_colocated_row_binlog, const Tablet* base_tablet, const std::vector& data_dirs, RuntimeProfile* profile); diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index 557fe02743b178..9746dbfe367631 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -128,7 +128,7 @@ TabletMetaSharedPtr TabletMeta::create( request.__isset.vertical_compaction_num_columns_per_group ? request.vertical_compaction_num_columns_per_group : 5, - request.__isset.row_binlog_schema ? &request.row_binlog_schema : nullptr); + request.__isset.is_row_binlog_tablet ? request.is_row_binlog_tablet : false); } TabletMeta::~TabletMeta() { @@ -140,8 +140,7 @@ TabletMeta::~TabletMeta() { TabletMeta::TabletMeta() : _tablet_uid(0, 0), _schema(new TabletSchema), - _delete_bitmap(new DeleteBitmap(_tablet_id)), - _binlog_delvec(new DeleteBitmap(_tablet_id)) {} + _delete_bitmap(new DeleteBitmap(_tablet_id)) {} TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id, int32_t schema_hash, int32_t shard_id, @@ -159,12 +158,10 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format, TEncryptionAlgorithm::type tde_algorithm, TStorageFormat::type storage_format, - int32_t vertical_compaction_num_columns_per_group, - const TTabletSchema* row_binlog_schema) + int32_t vertical_compaction_num_columns_per_group, bool is_row_binlog_tablet) : _tablet_uid(0, 0), _schema(new TabletSchema), _delete_bitmap(new DeleteBitmap(tablet_id)), - _binlog_delvec(new DeleteBitmap(tablet_id)), _storage_format(storage_format) { TabletMetaPB tablet_meta_pb; tablet_meta_pb.set_table_id(table_id); @@ -196,6 +193,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id time_series_compaction_level_threshold); tablet_meta_pb.set_vertical_compaction_num_columns_per_group( vertical_compaction_num_columns_per_group); + tablet_meta_pb.set_is_row_binlog_tablet(is_row_binlog_tablet); SchemaCreateOptions schema_create_options_for_data = { .col_ordinal_to_unique_id = col_ordinal_to_unique_id, .compression_type = compression_type, @@ -206,38 +204,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id tablet_meta_pb.set_in_restore_mode(false); - TabletSchemaPB* schema_pb_for_row_binlog = nullptr; - if (row_binlog_schema != nullptr) { - tablet_meta_pb.set_row_binlog_schema_hash(row_binlog_schema->schema_hash); - DCHECK(binlog_config.has_value()); - DCHECK(binlog_config->enable && binlog_config->binlog_format == TBinlogFormat::ROW); - - std::unordered_map row_binlog_col_ordinal_to_unique_id; - uint32_t row_binlog_next_unique_id = 0; - for (uint32_t col_ordinal = 0; col_ordinal < row_binlog_schema->columns.size(); - ++col_ordinal) { - const auto& tcolumn = row_binlog_schema->columns[col_ordinal]; - uint32_t unique_id = 0; - if (tcolumn.col_unique_id >= 0) { - unique_id = tcolumn.col_unique_id; - } else { - unique_id = col_ordinal; - } - row_binlog_col_ordinal_to_unique_id[col_ordinal] = unique_id; - if (row_binlog_next_unique_id <= unique_id) { - row_binlog_next_unique_id = unique_id + 1; - } - } - - SchemaCreateOptions schema_create_options_for_row_binlog = { - .col_ordinal_to_unique_id = row_binlog_col_ordinal_to_unique_id, - .compression_type = compression_type, - .inverted_index_file_storage_format = inverted_index_file_storage_format, - .next_unique_id = row_binlog_next_unique_id}; - schema_pb_for_row_binlog = tablet_meta_pb.mutable_row_binlog_schema(); - init_schema_from_thrift(*row_binlog_schema, schema_create_options_for_row_binlog, - schema_pb_for_row_binlog); - } if (binlog_config.has_value()) { BinlogConfig tmp_binlog_config; tmp_binlog_config = binlog_config.value(); @@ -266,10 +232,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id case TStorageFormat::V3: schema_pb_for_data->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); _schema->set_storage_format(TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); - if (schema_pb_for_row_binlog != nullptr) { - schema_pb_for_row_binlog->set_storage_format( - TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V3); - } break; default: break; @@ -301,11 +263,8 @@ TabletMeta::TabletMeta(const TabletMeta& b) _cooldown_meta_id(b._cooldown_meta_id), _enable_unique_key_merge_on_write(b._enable_unique_key_merge_on_write), _delete_bitmap(b._delete_bitmap), - _binlog_delvec(b._binlog_delvec), - _row_binlog_schema_hash(b._row_binlog_schema_hash), - _row_binlog_schema(b._row_binlog_schema), - _row_binlog_rs_metas(b._row_binlog_rs_metas), _binlog_config(b._binlog_config), + _is_row_binlog_tablet(b._is_row_binlog_tablet), _compaction_policy(b._compaction_policy), _time_series_compaction_goal_size_mbytes(b._time_series_compaction_goal_size_mbytes), _time_series_compaction_file_count_threshold( @@ -865,17 +824,9 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { _handle = pair.first; _schema = pair.second; - if (tablet_meta_pb.has_row_binlog_schema()) { - TabletSchemaSPtr row_binlog_schema = std::make_shared(); - row_binlog_schema->init_from_pb(tablet_meta_pb.row_binlog_schema()); - _row_binlog_schema = std::move(row_binlog_schema); - _row_binlog_schema_hash = tablet_meta_pb.row_binlog_schema_hash(); - } - if (tablet_meta_pb.has_enable_unique_key_merge_on_write()) { _enable_unique_key_merge_on_write = tablet_meta_pb.enable_unique_key_merge_on_write(); _delete_bitmap->set_tablet_id(_tablet_id); - _binlog_delvec->set_tablet_id(_tablet_id); } // init _rs_metas @@ -896,12 +847,6 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { } } - for (auto& it : tablet_meta_pb.row_binlog_rs_metas()) { - RowsetMetaSharedPtr rs_meta(new RowsetMeta()); - rs_meta->init_from_pb(it); - _row_binlog_rs_metas.emplace(rs_meta->version(), rs_meta); - } - if (tablet_meta_pb.has_in_restore_mode()) { _in_restore_mode = tablet_meta_pb.in_restore_mode(); } @@ -920,32 +865,22 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { int seg_ids_size = tablet_meta_pb.delete_bitmap().segment_ids_size(); int versions_size = tablet_meta_pb.delete_bitmap().versions_size(); int seg_maps_size = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps_size(); - int binlog_mark_size = tablet_meta_pb.delete_bitmap().is_binlog_delvec_size(); CHECK(rst_ids_size == seg_ids_size && seg_ids_size == seg_maps_size && seg_maps_size == versions_size); - CHECK(binlog_mark_size == 0 || binlog_mark_size == rst_ids_size); for (int i = 0; i < rst_ids_size; ++i) { RowsetId rst_id; rst_id.init(tablet_meta_pb.delete_bitmap().rowset_ids(i)); auto seg_id = tablet_meta_pb.delete_bitmap().segment_ids(i); auto ver = tablet_meta_pb.delete_bitmap().versions(i); auto bitmap = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps(i).data(); - bool from_binlog = tablet_meta_pb.delete_bitmap().is_binlog_delvec_size() > 0 - ? tablet_meta_pb.delete_bitmap().is_binlog_delvec(i) - : false; - if (!from_binlog) { - delete_bitmap().delete_bitmap[{rst_id, seg_id, ver}] = - roaring::Roaring::read(bitmap); - } else { - binlog_delvec().delete_bitmap[{rst_id, seg_id, ver}] = - roaring::Roaring::read(bitmap); - } + delete_bitmap().delete_bitmap[{rst_id, seg_id, ver}] = roaring::Roaring::read(bitmap); } } if (tablet_meta_pb.has_binlog_config()) { _binlog_config = tablet_meta_pb.binlog_config(); } + _is_row_binlog_tablet = tablet_meta_pb.is_row_binlog_tablet(); _compaction_policy = tablet_meta_pb.compaction_policy(); _time_series_compaction_goal_size_mbytes = tablet_meta_pb.time_series_compaction_goal_size_mbytes(); @@ -963,10 +898,6 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { if (tablet_meta_pb.has_encryption_algorithm()) { _encryption_algorithm = tablet_meta_pb.encryption_algorithm(); } - - if (tablet_meta_pb.has_row_binlog_schema_hash()) { - _row_binlog_schema_hash = tablet_meta_pb.row_binlog_schema_hash(); - } } void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_meta) { @@ -1008,18 +939,10 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_ for (const auto& [_, rs] : _stale_rs_metas) { rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas()); } - for (const auto& [_, rs] : _row_binlog_rs_metas) { - rs->to_rowset_pb(tablet_meta_pb->add_row_binlog_rs_metas()); - } } _schema->to_schema_pb(tablet_meta_pb->mutable_schema()); - if (_row_binlog_schema != nullptr) { - _row_binlog_schema->to_schema_pb(tablet_meta_pb->mutable_row_binlog_schema()); - tablet_meta_pb->set_row_binlog_schema_hash(_row_binlog_schema_hash); - } - tablet_meta_pb->set_in_restore_mode(in_restore_mode()); // to avoid modify tablet meta to the greatest extend @@ -1049,24 +972,13 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_ delete_bitmap_pb->add_rowset_ids(rowset_id.to_string()); delete_bitmap_pb->add_segment_ids(segment_id); delete_bitmap_pb->add_versions(ver); - delete_bitmap_pb->add_is_binlog_delvec(false); - std::string bitmap_data(bitmap.getSizeInBytes(), '\0'); - bitmap.write(bitmap_data.data()); - *(delete_bitmap_pb->add_segment_delete_bitmaps()) = std::move(bitmap_data); - } - - for (auto& [id, bitmap] : binlog_delvec().snapshot().delete_bitmap) { - auto& [rowset_id, segment_id, ver] = id; - delete_bitmap_pb->add_rowset_ids(rowset_id.to_string()); - delete_bitmap_pb->add_segment_ids(segment_id); - delete_bitmap_pb->add_versions(ver); - delete_bitmap_pb->add_is_binlog_delvec(true); std::string bitmap_data(bitmap.getSizeInBytes(), '\0'); bitmap.write(bitmap_data.data()); *(delete_bitmap_pb->add_segment_delete_bitmaps()) = std::move(bitmap_data); } } _binlog_config.to_pb(tablet_meta_pb->mutable_binlog_config()); + tablet_meta_pb->set_is_row_binlog_tablet(_is_row_binlog_tablet); tablet_meta_pb->set_compaction_policy(compaction_policy()); tablet_meta_pb->set_time_series_compaction_goal_size_mbytes( time_series_compaction_goal_size_mbytes()); @@ -1128,24 +1040,6 @@ Status TabletMeta::add_rs_meta(const RowsetMetaSharedPtr& rs_meta) { return Status::OK(); } -Status TabletMeta::add_row_binlog_rs_meta(const RowsetMetaSharedPtr& row_binlog_meta) { - // check RowsetMeta is valid - for (auto& [_, rs] : _row_binlog_rs_metas) { - if (rs->version() == row_binlog_meta->version()) { - if (rs->rowset_id() != row_binlog_meta->rowset_id()) { - return Status::Error( - "binlog version already exist. binlog_rowset_id={}, version={}, tablet={}", - rs->rowset_id().to_string(), rs->version().to_string(), tablet_id()); - } else { - // rowsetid,version is equal, it is a duplicate req, skip it - return Status::OK(); - } - } - } - _row_binlog_rs_metas.emplace(row_binlog_meta->version(), row_binlog_meta); - return Status::OK(); -} - void TabletMeta::add_rowsets_unchecked(const std::vector& to_add) { for (const auto& rs : to_add) { _rs_metas.emplace(rs->rowset_meta()->version(), rs->rowset_meta()); @@ -1197,17 +1091,6 @@ void TabletMeta::modify_rs_metas(const std::vector& to_add, _check_mow_rowset_cache_version_size(rowset_cache_version_size); } -void TabletMeta::modify_row_binlog_rs_metas(const std::vector& to_add, - const std::vector& to_delete) { - for (const auto& rs_to_del : to_delete) { - _row_binlog_rs_metas.erase(rs_to_del->version()); - } - - for (const auto& rs_to_add : to_add) { - _row_binlog_rs_metas.emplace(rs_to_add->version(), rs_to_add); - } -} - // Use the passing "rs_metas" to replace the rs meta in this tablet meta // Also clear the _stale_rs_metas because this tablet meta maybe copyied from // an existing tablet before. Add after revise, only the passing "rs_metas" @@ -1226,14 +1109,6 @@ void TabletMeta::revise_rs_metas(std::vector&& rs_metas) { } } -void TabletMeta::revise_row_binlog_rs_metas(std::vector&& rs_metas) { - std::lock_guard wrlock(_meta_lock); - _row_binlog_rs_metas.clear(); - for (auto& rs_meta : rs_metas) { - _row_binlog_rs_metas.emplace(rs_meta->version(), rs_meta); - } -} - // This method should call after revise_rs_metas, since new rs_metas might be a subset // of original tablet, we should revise the delete_bitmap according to current rowset. // @@ -1255,16 +1130,6 @@ void TabletMeta::revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap } } -void TabletMeta::revise_binlog_delvec_unlocked(const DeleteBitmap& binlog_delvec) { - _binlog_delvec = std::make_unique(tablet_id()); - for (const auto& [_, rs] : _row_binlog_rs_metas) { - DeleteBitmap rs_bm(tablet_id()); - binlog_delvec.subset({rs->rowset_id(), 0, 0}, {rs->rowset_id(), UINT32_MAX, INT64_MAX}, - &rs_bm); - _binlog_delvec->merge(rs_bm); - } -} - void TabletMeta::delete_stale_rs_meta_by_version(const Version& version) { _stale_rs_metas.erase(version); } @@ -1283,14 +1148,6 @@ RowsetMetaSharedPtr TabletMeta::acquire_stale_rs_meta_by_version(const Version& return nullptr; } -RowsetMetaSharedPtr TabletMeta::acquire_row_binlog_rs_meta_by_version( - const Version& version) const { - if (auto it = _row_binlog_rs_metas.find(version); it != _row_binlog_rs_metas.end()) { - return it->second; - } - return nullptr; -} - Status TabletMeta::set_partition_id(int64_t partition_id) { if ((_partition_id > 0 && _partition_id != partition_id) || partition_id < 1) { LOG(WARNING) << "cur partition id=" << _partition_id << " new partition id=" << partition_id diff --git a/be/src/storage/tablet/tablet_meta.h b/be/src/storage/tablet/tablet_meta.h index 5625a33c846ce3..73727e6f7cd104 100644 --- a/be/src/storage/tablet/tablet_meta.h +++ b/be/src/storage/tablet/tablet_meta.h @@ -119,7 +119,7 @@ class TabletMeta : public MetadataAdder { TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT, TStorageFormat::type storage_format = TStorageFormat::V2, int32_t vertical_compaction_num_columns_per_group = 5, - const TTabletSchema* row_binlog_schema = nullptr); + bool is_row_binlog_tablet = false); // If need add a filed in TableMeta, filed init copy in copy construct function TabletMeta(const TabletMeta& tablet_meta); TabletMeta(TabletMeta&& tablet_meta) = delete; @@ -174,8 +174,6 @@ class TabletMeta : public MetadataAdder { size_t tablet_local_size() const; // Remote disk space occupied by tablet. size_t tablet_remote_size() const; - size_t binlog_size() const; - size_t tablet_local_index_size() const; size_t tablet_local_segment_size() const; size_t tablet_remote_index_size() const; @@ -183,7 +181,6 @@ class TabletMeta : public MetadataAdder { size_t version_count() const; size_t stale_version_count() const; - size_t binlog_file_num() const; size_t version_count_cross_with_range(const Version& range) const; Version max_version() const; @@ -207,13 +204,8 @@ class TabletMeta : public MetadataAdder { void modify_rs_metas(const std::vector& to_add, const std::vector& to_delete, bool same_version = false); - void modify_row_binlog_rs_metas(const std::vector& to_add, - const std::vector& to_delete); void revise_rs_metas(std::vector&& rs_metas); - void revise_row_binlog_rs_metas(std::vector&& rs_metas); void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap); - // Revise delete vector used by binlog rowsets. - void revise_binlog_delvec_unlocked(const DeleteBitmap& binlog_delvec); const RowsetMetaMapContainer& all_stale_rs_metas() const; RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version) const; @@ -267,9 +259,6 @@ class TabletMeta : public MetadataAdder { DeleteBitmapPtr delete_bitmap_ptr() { return _delete_bitmap; } DeleteBitmap& delete_bitmap() { return *_delete_bitmap; } - DeleteBitmapPtr binlog_delvec_ptr() { return _binlog_delvec; } - DeleteBitmap& binlog_delvec() { return *_binlog_delvec; } - void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version); bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } @@ -283,12 +272,10 @@ class TabletMeta : public MetadataAdder { void set_binlog_config(BinlogConfig binlog_config) { _binlog_config = std::move(binlog_config); } - - const TabletSchemaSPtr& row_binlog_schema() const { return _row_binlog_schema; } - int32_t row_binlog_schema_hash() const { return _row_binlog_schema_hash; } - const RowsetMetaMapContainer& all_row_binlog_rs_metas() const; - RowsetMetaSharedPtr acquire_row_binlog_rs_meta_by_version(const Version& version) const; - Status add_row_binlog_rs_meta(const RowsetMetaSharedPtr& rs_meta); + bool is_row_binlog_tablet() const { return _is_row_binlog_tablet; } + void set_is_row_binlog_tablet(bool is_row_binlog_tablet) { + _is_row_binlog_tablet = is_row_binlog_tablet; + } void set_compaction_policy(std::string compaction_policy) { _compaction_policy = compaction_policy; @@ -391,14 +378,10 @@ class TabletMeta : public MetadataAdder { // query performance significantly. bool _enable_unique_key_merge_on_write = false; std::shared_ptr _delete_bitmap; - std::shared_ptr _binlog_delvec; - - int32_t _row_binlog_schema_hash = 0; - TabletSchemaSPtr _row_binlog_schema; - RowsetMetaMapContainer _row_binlog_rs_metas; // binlog config BinlogConfig _binlog_config {}; + bool _is_row_binlog_tablet = false; // meta for compaction std::string _compaction_policy; @@ -751,8 +734,6 @@ inline size_t TabletMeta::tablet_local_size() const { total_size += rs->total_disk_size(); } } - // if we need to split data and binlog or not - total_size += binlog_size(); return total_size; } @@ -766,16 +747,6 @@ inline size_t TabletMeta::tablet_remote_size() const { return total_size; } -inline size_t TabletMeta::binlog_size() const { - size_t total_size = 0; - for (auto& [_, rs] : _row_binlog_rs_metas) { - if (rs->is_local()) { - total_size += rs->data_disk_size(); - } - } - return total_size; -} - inline size_t TabletMeta::tablet_local_index_size() const { size_t total_size = 0; for (const auto& [_, rs] : _rs_metas) { @@ -824,10 +795,6 @@ inline size_t TabletMeta::stale_version_count() const { return _rs_metas.size(); } -inline size_t TabletMeta::binlog_file_num() const { - return _row_binlog_rs_metas.size(); -} - inline TabletState TabletMeta::tablet_state() const { return _tablet_state; } @@ -864,10 +831,6 @@ inline const RowsetMetaMapContainer& TabletMeta::all_stale_rs_metas() const { return _stale_rs_metas; } -inline const RowsetMetaMapContainer& TabletMeta::all_row_binlog_rs_metas() const { - return _row_binlog_rs_metas; -} - inline bool TabletMeta::all_beta() const { for (const auto& [_, rs] : _rs_metas) { if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) { @@ -879,11 +842,6 @@ inline bool TabletMeta::all_beta() const { return false; } } - for (const auto& [_, rs] : _row_binlog_rs_metas) { - if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) { - return false; - } - } return true; } diff --git a/be/src/storage/tablet/tablet_meta_manager.cpp b/be/src/storage/tablet/tablet_meta_manager.cpp index 45815a292aa96e..0f43dbd409e49b 100644 --- a/be/src/storage/tablet/tablet_meta_manager.cpp +++ b/be/src/storage/tablet/tablet_meta_manager.cpp @@ -235,46 +235,21 @@ void NO_SANITIZE_UNDEFINED TabletMetaManager::decode_delete_bitmap_key(std::stri Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id, DeleteBitmapPtr delete_bitmap, int64_t version) { - return save_delete_bitmap(store, tablet_id, std::move(delete_bitmap), nullptr, version); -} - -Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id, - DeleteBitmapPtr delete_bitmap, - DeleteBitmapPtr binlog_delvec, int64_t version) { VLOG_NOTICE << "save delete bitmap, tablet_id:" << tablet_id << ", version: " << version; - if ((delete_bitmap == nullptr || delete_bitmap->delete_bitmap.empty()) && - (binlog_delvec == nullptr || binlog_delvec->delete_bitmap.empty())) { + if (delete_bitmap == nullptr || delete_bitmap->delete_bitmap.empty()) { return Status::OK(); } OlapMeta* meta = store->get_meta(); DeleteBitmapPB delete_bitmap_pb; - // Normal delete bitmap. - if (delete_bitmap != nullptr) { - for (auto& [id, bitmap] : delete_bitmap->delete_bitmap) { - auto& rowset_id = std::get<0>(id); - auto segment_id = std::get<1>(id); - delete_bitmap_pb.add_rowset_ids(rowset_id.to_string()); - delete_bitmap_pb.add_segment_ids(segment_id); - std::string bitmap_data(bitmap.getSizeInBytes(), '\0'); - bitmap.write(bitmap_data.data()); - *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data); - delete_bitmap_pb.add_is_binlog_delvec(false); - } - } - - // Binlog delvec. - if (binlog_delvec != nullptr) { - for (auto& [id, bitmap] : binlog_delvec->delete_bitmap) { - auto& rowset_id = std::get<0>(id); - auto segment_id = std::get<1>(id); - delete_bitmap_pb.add_rowset_ids(rowset_id.to_string()); - delete_bitmap_pb.add_segment_ids(segment_id); - std::string bitmap_data(bitmap.getSizeInBytes(), '\0'); - bitmap.write(bitmap_data.data()); - *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data); - delete_bitmap_pb.add_is_binlog_delvec(true); - } + for (auto& [id, bitmap] : delete_bitmap->delete_bitmap) { + auto& rowset_id = std::get<0>(id); + auto segment_id = std::get<1>(id); + delete_bitmap_pb.add_rowset_ids(rowset_id.to_string()); + delete_bitmap_pb.add_segment_ids(segment_id); + std::string bitmap_data(bitmap.getSizeInBytes(), '\0'); + bitmap.write(bitmap_data.data()); + *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data); } std::string key = encode_delete_bitmap_key(tablet_id, version); diff --git a/be/src/storage/tablet/tablet_meta_manager.h b/be/src/storage/tablet/tablet_meta_manager.h index c0676a7ef6c5d1..eafcce9191191f 100644 --- a/be/src/storage/tablet/tablet_meta_manager.h +++ b/be/src/storage/tablet/tablet_meta_manager.h @@ -75,12 +75,6 @@ class TabletMetaManager { static Status save_delete_bitmap(DataDir* store, TTabletId tablet_id, DeleteBitmapPtr delete_bitmap, int64_t version); - // Persist both normal delete bitmap and binlog delvec for a specific visible version. - // `binlog_delvec` is optional. - static Status save_delete_bitmap(DataDir* store, TTabletId tablet_id, - DeleteBitmapPtr delete_bitmap, DeleteBitmapPtr binlog_delvec, - int64_t version); - static Status traverse_delete_bitmap( OlapMeta* meta, std::function const& func); diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index 3feb0d9009bcfd..5eccf8df205f0c 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -139,8 +139,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { } bool need_ordered_result = true; - if (read_params.reader_type == ReaderType::READER_QUERY || - read_params.reader_type == ReaderType::READER_BINLOG) { + if (read_params.reader_type == ReaderType::READER_QUERY) { if (_tablet_schema->keys_type() == DUP_KEYS) { // duplicated keys are allowed, no need to merge sort keys in rowset need_ordered_result = false; @@ -168,6 +167,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { } _reader_context.reader_type = read_params.reader_type; + _reader_context.read_row_binlog = read_params.read_row_binlog; _reader_context.version = read_params.version; _reader_context.tablet_schema = _tablet_schema; _reader_context.need_ordered_result = need_ordered_result; @@ -175,9 +175,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.topn_filter_target_node_id = read_params.topn_filter_target_node_id; _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; _reader_context.use_insert_order_when_same = - read_params.use_insert_order_when_same || - read_params.reader_type == ReaderType::READER_BINLOG || - read_params.reader_type == ReaderType::READER_BINLOG_COMPACTION; + read_params.use_insert_order_when_same || read_params.read_row_binlog; _reader_context.force_key_ordered_read = read_params.force_key_ordered_read; _reader_context.read_orderby_key_limit = read_params.read_orderby_key_limit; _reader_context.return_columns = &_return_columns; @@ -298,8 +296,7 @@ Status TabletReader::_init_params(const ReaderParams& read_params) { Status TabletReader::_init_return_columns(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns); - if (read_params.reader_type == ReaderType::READER_QUERY || - read_params.reader_type == ReaderType::READER_BINLOG) { + if (read_params.reader_type == ReaderType::READER_QUERY) { _return_columns = read_params.return_columns; _tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set; for (auto id : read_params.return_columns) { @@ -323,7 +320,6 @@ Status TabletReader::_init_return_columns(const ReaderParams& read_params) { read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION || read_params.reader_type == ReaderType::READER_BASE_COMPACTION || read_params.reader_type == ReaderType::READER_FULL_COMPACTION || - read_params.reader_type == ReaderType::READER_BINLOG_COMPACTION || read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || read_params.reader_type == ReaderType::READER_ALTER_TABLE) && !read_params.return_columns.empty()) { diff --git a/be/src/storage/tablet/tablet_reader.h b/be/src/storage/tablet/tablet_reader.h index b916721588a98f..e8350639d02994 100644 --- a/be/src/storage/tablet/tablet_reader.h +++ b/be/src/storage/tablet/tablet_reader.h @@ -121,7 +121,7 @@ class TabletReader { rs_splits = std::move(read_source.rs_splits); delete_predicates = std::move(read_source.delete_predicates); #ifndef BE_TEST - if (tablet->enable_unique_key_merge_on_write() && !skip_delete_bitmap) { + if (!skip_delete_bitmap && tablet->need_read_delete_bitmap()) { delete_bitmap = std::move(read_source.delete_bitmap); } #endif @@ -130,6 +130,7 @@ class TabletReader { BaseTabletSPtr tablet; TabletSchemaSPtr tablet_schema; ReaderType reader_type = ReaderType::READER_QUERY; + bool read_row_binlog = false; bool direct_mode = false; bool aggregation = false; // for compaction, schema_change, check_sum: we don't use page cache diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index caa7c9b54f6c42..a39a1b22b1ce6d 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -518,6 +518,7 @@ class TabletSchema : public MetadataAdder { int32_t version_col_idx() const { return _version_col_idx; } bool has_skip_bitmap_col() const { return _skip_bitmap_col_idx != -1; } int32_t skip_bitmap_col_idx() const { return _skip_bitmap_col_idx; } + bool enable_tso() const { return _commit_tso_col_idx != -1 || _binlog_tso_col_idx != -1; } int32_t commit_tso_col_idx() const { return _commit_tso_col_idx; } int32_t binlog_tso_col_idx() const { return _binlog_tso_col_idx; } int32_t binlog_lsn_col_idx() const { return _binlog_lsn_col_idx; } diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index 88d2c79b3ed268..9fee41082442c2 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -60,6 +60,15 @@ namespace doris { +const OlapTableIndexSchema* OlapTableSchemaParam::row_binlog_index_schema(int64_t index_id) const { + for (auto* schema : _row_binlog_index_schemas) { + if (schema->index_id == index_id) { + return schema; + } + } + return nullptr; +} + void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { pindex->set_id(index_id); pindex->set_schema_hash(schema_hash); @@ -225,8 +234,7 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { _indexes.emplace_back(index); } - if (pschema.has_row_binlog_index_schema()) { - const auto& p_index = pschema.row_binlog_index_schema(); + for (const auto& p_index : pschema.row_binlog_index_schemas()) { auto* index = _obj_pool.add(new OlapTableIndexSchema()); index->index_id = p_index.id(); index->schema_hash = p_index.schema_hash(); @@ -243,7 +251,7 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { ti->init_from_pb(pindex_desc); index->indexes.emplace_back(ti); } - _row_binlog_index_schema = index; + _row_binlog_index_schemas.emplace_back(index); } std::sort(_indexes.begin(), _indexes.end(), @@ -402,20 +410,21 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { _indexes.emplace_back(index); } - if (tschema.__isset.row_binlog_index_schema) { - const auto& t_index = tschema.row_binlog_index_schema; - auto* index = _obj_pool.add(new OlapTableIndexSchema()); - index->index_id = t_index.id; - index->schema_hash = t_index.schema_hash; - if (t_index.__isset.row_binlog_id) { - index->row_binlog_id = t_index.row_binlog_id; - } - for (const auto& tcolumn_desc : t_index.columns_desc) { - TabletColumn* tc = _obj_pool.add(new TabletColumn()); - tc->init_from_thrift(tcolumn_desc); - index->columns.emplace_back(tc); + if (tschema.__isset.row_binlog_index_schemas) { + for (const auto& t_index : tschema.row_binlog_index_schemas) { + auto* index = _obj_pool.add(new OlapTableIndexSchema()); + index->index_id = t_index.id; + index->schema_hash = t_index.schema_hash; + if (t_index.__isset.row_binlog_id) { + index->row_binlog_id = t_index.row_binlog_id; + } + for (const auto& tcolumn_desc : t_index.columns_desc) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_thrift(tcolumn_desc); + index->columns.emplace_back(tc); + } + _row_binlog_index_schemas.emplace_back(index); } - _row_binlog_index_schema = index; } std::sort(_indexes.begin(), _indexes.end(), @@ -452,8 +461,8 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const { for (auto* index : _indexes) { index->to_protobuf(pschema->add_indexes()); } - if (_row_binlog_index_schema != nullptr) { - _row_binlog_index_schema->to_protobuf(pschema->mutable_row_binlog_index_schema()); + for (auto* index : _row_binlog_index_schemas) { + index->to_protobuf(pschema->add_row_binlog_index_schemas()); } } diff --git a/be/src/storage/tablet_info.h b/be/src/storage/tablet_info.h index aeb78badf4c425..1ea346844d89d6 100644 --- a/be/src/storage/tablet_info.h +++ b/be/src/storage/tablet_info.h @@ -34,6 +34,7 @@ #include #include +#include "cloud/config.h" #include "common/cast_set.h" #include "common/logging.h" #include "common/object_pool.h" @@ -80,7 +81,10 @@ class OlapTableSchemaParam { TupleDescriptor* tuple_desc() const { return _tuple_desc; } const std::vector& indexes() const { return _indexes; } - const OlapTableIndexSchema* row_binlog_index_schema() const { return _row_binlog_index_schema; } + const OlapTableIndexSchema* row_binlog_index_schema(int64_t index_id) const; + const std::vector& row_binlog_index_schemas() const { + return _row_binlog_index_schemas; + } void to_protobuf(POlapTableSchemaParam* pschema) const; @@ -133,7 +137,7 @@ class OlapTableSchemaParam { TupleDescriptor* _tuple_desc = nullptr; mutable POlapTableSchemaParam* _proto_schema = nullptr; std::vector _indexes; - OlapTableIndexSchema* _row_binlog_index_schema = nullptr; + std::vector _row_binlog_index_schemas; mutable ObjectPool _obj_pool; UniqueKeyUpdateModePB _unique_key_update_mode {UniqueKeyUpdateModePB::UPSERT}; PartialUpdateNewRowPolicyPB _partial_update_new_row_policy { @@ -374,6 +378,7 @@ class OlapTableLocationParam { OlapTableLocationParam(const TOlapTableLocationParam& t_param) : _t_param(t_param) { for (auto& location : _t_param.tablets) { _tablets.emplace(location.tablet_id, &location); + _update_base_to_binlog_tablet(location); } } @@ -389,18 +394,48 @@ class OlapTableLocationParam { return nullptr; } + // used by base tablet on node_id to mark whether to write binlog, 0 means no binlog. + int64_t get_binlog_tablet_id(int64_t base_tablet_id, int64_t node_id) const { + auto it = _base_to_binlog_tablet.find({base_tablet_id, node_id}); + if (it != _base_to_binlog_tablet.end()) { + return it->second; + } + if (config::is_cloud_mode()) { + auto cloud_it = _base_to_binlog_tablet.lower_bound({base_tablet_id, INT64_MIN}); + if (cloud_it != _base_to_binlog_tablet.end() && + cloud_it->first.first == base_tablet_id) { + return cloud_it->second; + } + } + return 0; + } + void add_locations(std::vector& locations) { for (auto& location : locations) { if (_tablets.find(location.tablet_id) == _tablets.end()) { _tablets[location.tablet_id] = &location; } + _update_base_to_binlog_tablet(location); } } private: + void _update_base_to_binlog_tablet(const TTabletLocation& location) { + if (!location.__isset.base_tablet_id) { + return; + } + // the binlog tablet is written by its base tablet on the nodes that own both. + for (int64_t node_id : location.node_ids) { + _base_to_binlog_tablet.emplace(std::make_pair(location.base_tablet_id, node_id), + location.tablet_id); + } + } + TOlapTableLocationParam _t_param; // [tablet_id, tablet]. tablet has id, also. std::unordered_map _tablets; + // [(base_tablet_id, node_id), row_binlog_tablet_id] + std::map, int64_t> _base_to_binlog_tablet; }; struct NodeInfo { diff --git a/be/src/storage/task/engine_clone_task.cpp b/be/src/storage/task/engine_clone_task.cpp index e6fe3d3031d352..5425a2f661d20f 100644 --- a/be/src/storage/task/engine_clone_task.cpp +++ b/be/src/storage/task/engine_clone_task.cpp @@ -181,10 +181,6 @@ Status EngineCloneTask::_do_clone() { Status status = Status::OK(); std::string src_file_path; TBackend src_host; - int32_t copy_type = - _clone_req.__isset.copy_type ? _clone_req.copy_type : TabletCopyType::DEFAULT; - RETURN_IF_ERROR(TabletCopyType::validate(copy_type)); - bool copy_row_binlog = TabletCopyType::has(copy_type, TTabletCopyType::ROW_BINLOG); RETURN_IF_ERROR( _engine.tablet_manager()->register_transition_tablet(_clone_req.tablet_id, "clone")); Defer defer {[&]() { @@ -259,7 +255,7 @@ Status EngineCloneTask::_do_clone() { &src_host, &src_file_path, missed_versions, &allow_incremental_clone)); RETURN_IF_ERROR(_finish_clone(tablet.get(), local_data_path, specified_version, - allow_incremental_clone, copy_row_binlog)); + allow_incremental_clone)); } else { LOG(INFO) << "clone tablet not exist, begin clone a new tablet from remote be. " << "signature=" << _signature << ", tablet_id=" << _clone_req.tablet_id @@ -426,11 +422,6 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, } std::string address = get_host_port(src.host, src.http_port); - int32_t copy_type = - _clone_req.__isset.copy_type ? _clone_req.copy_type : TabletCopyType::DEFAULT; - RETURN_IF_ERROR(TabletCopyType::validate(copy_type)); - bool copy_row_binlog = TabletCopyType::has(copy_type, TTabletCopyType::ROW_BINLOG); - if (config::enable_batch_download && is_support_batch_download(address).ok()) { // download files via batch api. LOG_INFO("remote BE supports batch download, use batch file download") @@ -444,21 +435,6 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, .error(status); continue; // Try another BE } - if (copy_row_binlog) { - std::string row_binlog_remote_dir = - fmt::format("{}{}/", remote_dir, FDRowBinlogSuffix); - std::string row_binlog_local_path = - fmt::format("{}/{}", local_data_path, FDRowBinlogSuffix); - status = _batch_download_files(&data_dir, address, row_binlog_remote_dir, - row_binlog_local_path); - if (!status.ok()) [[unlikely]] { - LOG_WARNING("failed to download row binlog snapshot from remote BE in batch") - .tag("address", address) - .tag("remote_dir", row_binlog_remote_dir) - .error(status); - continue; // Try another BE - } - } } else { if (config::enable_batch_download) { LOG_INFO("remote BE does not support batch download, use single file download") @@ -485,26 +461,6 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, .error(status); continue; // Try another BE } - if (copy_row_binlog) { - std::string row_binlog_remote_url_prefix; - { - std::stringstream ss; - ss << "http://" << address << HTTP_REQUEST_PREFIX << HTTP_REQUEST_TOKEN_PARAM - << token << HTTP_REQUEST_FILE_PARAM << remote_dir << FDRowBinlogSuffix - << "/"; - row_binlog_remote_url_prefix = ss.str(); - } - std::string row_binlog_local_path = - fmt::format("{}/{}", local_data_path, FDRowBinlogSuffix); - status = _download_files(&data_dir, row_binlog_remote_url_prefix, - row_binlog_local_path); - if (!status.ok()) [[unlikely]] { - LOG_WARNING("failed to download row binlog snapshot from remote BE") - .tag("url", mask_token(row_binlog_remote_url_prefix)) - .error(status); - continue; // Try another BE - } - } } // No need to try again with another BE @@ -525,11 +481,7 @@ Status EngineCloneTask::_make_snapshot(const std::string& ip, int port, TTableId request.__set_schema_hash(schema_hash); request.__set_preferred_snapshot_version(g_Types_constants.TPREFER_SNAPSHOT_REQ_VERSION); request.__set_version(_clone_req.version); - int32_t copy_type = - _clone_req.__isset.copy_type ? _clone_req.copy_type : TabletCopyType::DEFAULT; - RETURN_IF_ERROR(TabletCopyType::validate(copy_type)); - request.__set_copy_type(copy_type); - request.__set_is_copy_binlog(TabletCopyType::has(copy_type, TTabletCopyType::CCR_BINLOG)); + request.__set_is_copy_binlog(true); // TODO: missing version composed of singleton delta. // if not, this place should be rewrote. // we make every TSnapshotRequest sent from be with __isset.missing_version = true @@ -773,7 +725,7 @@ Status EngineCloneTask::_batch_download_files(DataDir* data_dir, const std::stri /// 1. Link all files from CLONE dir to tablet dir if file does not exist in tablet dir /// 2. Call _finish_xx_clone() to revise the tablet meta. Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_dir, int64_t version, - bool is_incremental_clone, bool copy_row_binlog) { + bool is_incremental_clone) { Defer remove_clone_dir {[&]() { std::error_code ec; std::filesystem::remove_all(clone_dir, ec); @@ -829,7 +781,7 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(binlog_dir)); } - enum class CloneFileType { DATA, ROW_BINLOG, CCR_BINLOG }; + enum class CloneFileType { DATA, CCR_BINLOG }; // check all files in /clone and /tablet std::vector clone_files; @@ -842,38 +794,12 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d } clone_file_names.emplace_back(file_type, file.file_name); } - auto row_binlog_clone_dir = fmt::format("{}/{}", clone_dir, FDRowBinlogSuffix); - if (copy_row_binlog) { - clone_files.clear(); - RETURN_IF_ERROR(io::global_local_filesystem()->list(row_binlog_clone_dir, true, - &clone_files, &exists)); - if (!exists) { - return Status::InternalError("row binlog clone dir not existed. clone_dir={}", - row_binlog_clone_dir); - } - for (auto& file : clone_files) { - clone_file_names.emplace_back(CloneFileType::ROW_BINLOG, file.file_name); - } - } - std::vector local_files; RETURN_IF_ERROR(io::global_local_filesystem()->list(tablet_dir, true, &local_files, &exists)); std::unordered_set data_local_file_names; for (auto& file : local_files) { data_local_file_names.insert(file.file_name); } - auto row_binlog_dir = tablet->row_binlog_path(); - std::unordered_set row_binlog_local_file_names; - if (copy_row_binlog) { - RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(row_binlog_dir)); - local_files.clear(); - RETURN_IF_ERROR( - io::global_local_filesystem()->list(row_binlog_dir, true, &local_files, &exists)); - for (auto& file : local_files) { - row_binlog_local_file_names.insert(file.file_name); - } - } - Status status; std::vector linked_success_files; Defer remove_linked_files {[&]() { // clear linked files if errors happen @@ -895,13 +821,6 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d << "tablet=" << tablet->tablet_id() << ", clone_file=" << clone_file; continue; } - if (clone_file_type == CloneFileType::ROW_BINLOG && - row_binlog_local_file_names.find(clone_file) != row_binlog_local_file_names.end()) { - VLOG_NOTICE << "find same row binlog file when clone, skip it. " - << "tablet=" << tablet->tablet_id() << ", clone_file=" << clone_file; - continue; - } - /// if binlog exist in clone dir and md5sum equal, then skip link file bool skip_link_file = false; std::string to; @@ -921,15 +840,11 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d return status; } } else { - auto& local_dir = - clone_file_type == CloneFileType::ROW_BINLOG ? row_binlog_dir : tablet_dir; - to = fmt::format("{}/{}", local_dir, clone_file); + to = fmt::format("{}/{}", tablet_dir, clone_file); } if (!skip_link_file) { - auto& clone_file_dir = - clone_file_type == CloneFileType::ROW_BINLOG ? row_binlog_clone_dir : clone_dir; - auto from = fmt::format("{}/{}", clone_file_dir, clone_file); + auto from = fmt::format("{}/{}", clone_dir, clone_file); status = io::global_local_filesystem()->link_file(from, to); if (!status.ok()) { return status; @@ -954,9 +869,9 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d std::lock_guard wrlock(tablet->get_header_lock()); SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD); if (is_incremental_clone) { - status = _finish_incremental_clone(tablet, cloned_tablet_meta, version, copy_row_binlog); + status = _finish_incremental_clone(tablet, cloned_tablet_meta, version); } else { - status = _finish_full_clone(tablet, cloned_tablet_meta, copy_row_binlog); + status = _finish_full_clone(tablet, cloned_tablet_meta); } // if full clone success, need to update cumulative layer point @@ -973,7 +888,7 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d /// 2. Revise the local tablet meta to add all incremental cloned rowset's meta. Status EngineCloneTask::_finish_incremental_clone(Tablet* tablet, const TabletMetaSharedPtr& cloned_tablet_meta, - int64_t version, bool copy_row_binlog) { + int64_t version) { LOG(INFO) << "begin to finish incremental clone. tablet=" << tablet->tablet_id() << ", visible_version=" << version << ", cloned_tablet_replica_id=" << cloned_tablet_meta->replica_id(); @@ -996,34 +911,19 @@ Status EngineCloneTask::_finish_incremental_clone(Tablet* tablet, RowsetSharedPtr rs; RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs)); rowsets_to_clone.push_back(std::move(rs)); - if (copy_row_binlog) { - rs_meta = cloned_tablet_meta->acquire_row_binlog_rs_meta_by_version(nested_version); - if (rs_meta == nullptr) { - return Status::InternalError( - "missed version {} row binlog is not found in cloned tablet meta", - nested_version.to_string()); - } - RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs)); - rowsets_to_clone.push_back(std::move(rs)); - } - } - - if (copy_row_binlog && tablet->enable_unique_key_merge_on_write()) { - tablet->tablet_meta()->binlog_delvec().merge(cloned_tablet_meta->binlog_delvec()); } /// clone_data to tablet /// For incremental clone, nothing will be deleted. /// So versions_to_delete is empty. - return tablet->revise_tablet_meta(rowsets_to_clone, {}, true, copy_row_binlog); + return tablet->revise_tablet_meta(rowsets_to_clone, {}, true); } /// This method will do: /// 1. Compare the version of local tablet and cloned tablet to decide which version to keep /// 2. Revise the local tablet meta Status EngineCloneTask::_finish_full_clone(Tablet* tablet, - const TabletMetaSharedPtr& cloned_tablet_meta, - bool copy_row_binlog) { + const TabletMetaSharedPtr& cloned_tablet_meta) { Version cloned_max_version = cloned_tablet_meta->max_version(); LOG(INFO) << "begin to finish full clone. tablet=" << tablet->tablet_id() << ", cloned_max_version=" << cloned_max_version; @@ -1061,36 +961,12 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, DCHECK(rs->is_local()); } } - if (copy_row_binlog) { - for (auto& [v, rs] : tablet->row_binlog_rowset_map()) { - if (v.first <= cloned_max_version.second && v.second > cloned_max_version.second) { - return Status::InternalError( - "row binlog version cross src latest. cloned_max_version={}, " - "local_version={}", - cloned_max_version.second, v.to_string()); - } - if (v.second <= cloned_max_version.second) { - to_delete.push_back(rs); - } else { - DCHECK(rs->is_local()); - } - } - } - - to_add.reserve(cloned_tablet_meta->all_rs_metas().size() + - cloned_tablet_meta->all_row_binlog_rs_metas().size()); + to_add.reserve(cloned_tablet_meta->all_rs_metas().size()); for (const auto& [_, rs_meta] : cloned_tablet_meta->all_rs_metas()) { RowsetSharedPtr rs; RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs)); to_add.push_back(std::move(rs)); } - if (copy_row_binlog) { - for (const auto& [_, rs_meta] : cloned_tablet_meta->all_row_binlog_rs_metas()) { - RowsetSharedPtr rs; - RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs)); - to_add.push_back(std::move(rs)); - } - } { std::shared_lock cooldown_conf_rlock(tablet->get_cooldown_conf_lock()); if (tablet->cooldown_conf_unlocked().cooldown_replica_id == tablet->replica_id()) { @@ -1109,11 +985,8 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, } if (tablet->enable_unique_key_merge_on_write()) { tablet->tablet_meta()->delete_bitmap().merge(cloned_tablet_meta->delete_bitmap()); - if (copy_row_binlog) { - tablet->tablet_meta()->binlog_delvec().merge(cloned_tablet_meta->binlog_delvec()); - } } - return tablet->revise_tablet_meta(to_add, to_delete, false, copy_row_binlog); + return tablet->revise_tablet_meta(to_add, to_delete, false); // TODO(plat1ko): write cooldown meta to remote if this replica is cooldown replica } } // namespace doris diff --git a/be/src/storage/task/engine_clone_task.h b/be/src/storage/task/engine_clone_task.h index 8189fcfbcf8b15..c26350e9ae673f 100644 --- a/be/src/storage/task/engine_clone_task.h +++ b/be/src/storage/task/engine_clone_task.h @@ -64,13 +64,12 @@ class EngineCloneTask final : public EngineTask { Status _do_clone(); Status _finish_clone(Tablet* tablet, const std::string& clone_dir, int64_t version, - bool is_incremental_clone, bool copy_row_binlog); + bool is_incremental_clone); Status _finish_incremental_clone(Tablet* tablet, const TabletMetaSharedPtr& cloned_tablet_meta, - int64_t version, bool copy_row_binlog); + int64_t version); - Status _finish_full_clone(Tablet* tablet, const TabletMetaSharedPtr& cloned_tablet_meta, - bool copy_row_binlog); + Status _finish_full_clone(Tablet* tablet, const TabletMetaSharedPtr& cloned_tablet_meta); Status _make_and_download_snapshots(DataDir& data_dir, const std::string& local_data_path, TBackend* src_host, std::string* src_file_path, diff --git a/be/src/storage/task/engine_publish_version_task.cpp b/be/src/storage/task/engine_publish_version_task.cpp index 240cef62d0edf1..aacd8bdc84d2c1 100644 --- a/be/src/storage/task/engine_publish_version_task.cpp +++ b/be/src/storage/task/engine_publish_version_task.cpp @@ -154,9 +154,9 @@ Status EnginePublishVersionTask::execute() { } map tablet_related_rs; - map> tablet_related_attach_rowsets; + map> tablet_related_txn_infos; _engine.txn_manager()->get_txn_related_tablets( - transaction_id, partition_id, &tablet_related_rs, &tablet_related_attach_rowsets); + transaction_id, partition_id, &tablet_related_rs, &tablet_related_txn_infos); Version version(par_ver_info.version, par_ver_info.version); @@ -170,8 +170,8 @@ Status EnginePublishVersionTask::execute() { for (auto& tablet_rs : tablet_related_rs) { TabletInfo tablet_info = tablet_rs.first; RowsetSharedPtr rowset = tablet_rs.second; - auto attach_rowsets_it = tablet_related_attach_rowsets.find(tablet_info); - DCHECK(attach_rowsets_it != tablet_related_attach_rowsets.end()); + auto txn_info_it = tablet_related_txn_infos.find(tablet_info); + DCHECK(txn_info_it != tablet_related_txn_infos.end()); VLOG_CRITICAL << "begin to publish version on tablet. " << "tablet_id=" << tablet_info.tablet_id << ", version=" << version.first << ", transaction_id=" << transaction_id; @@ -250,8 +250,8 @@ Status EnginePublishVersionTask::execute() { } auto tablet_publish_txn_ptr = std::make_shared( - _engine, this, tablet, rowset, attach_rowsets_it->second, partition_id, - transaction_id, version, tablet_info, par_ver_info.commit_tso); + _engine, this, tablet, rowset, txn_info_it->second->attach_row_binlog, + partition_id, transaction_id, version, tablet_info, par_ver_info.commit_tso); tablet_tasks.push_back(tablet_publish_txn_ptr); auto submit_st = token->submit_func([=]() { tablet_publish_txn_ptr->handle(); }); #ifndef NDEBUG @@ -409,13 +409,13 @@ void EnginePublishVersionTask::_calculate_tbl_num_delta_rows( TabletPublishTxnTask::TabletPublishTxnTask( StorageEngine& engine, EnginePublishVersionTask* engine_task, TabletSharedPtr tablet, - RowsetSharedPtr rowset, std::vector attach_rowsets, int64_t partition_id, + RowsetSharedPtr rowset, const RowBinlogTxnInfo& attach_row_binlog, int64_t partition_id, int64_t transaction_id, Version version, const TabletInfo& tablet_info, int64_t commit_tso) : _engine(engine), _engine_publish_version_task(engine_task), _tablet(std::move(tablet)), _rowset(std::move(rowset)), - _attach_rowsets(std::move(attach_rowsets)), + _attach_row_binlog(attach_row_binlog), _partition_id(partition_id), _transaction_id(transaction_id), _version(version), @@ -433,7 +433,6 @@ TabletPublishTxnTask::~TabletPublishTxnTask() = default; Status publish_version_and_add_rowset(StorageEngine& engine, int64_t partition_id, const TabletSharedPtr& tablet, const RowsetSharedPtr& rowset, - const std::vector& attach_rowsets, int64_t transaction_id, const Version& version, EnginePublishVersionTask* engine_publish_version_task, TabletPublishStatistics& stats, int64_t commit_tso) { @@ -458,38 +457,80 @@ Status publish_version_and_add_rowset(StorageEngine& engine, int64_t partition_i DBUG_EXECUTE_IF("EnginePublishVersionTask.handle.block_add_rowsets", DBUG_BLOCK); // Add visible rowset to tablet - int64_t start_time = MonotonicMicros(); - RowsetSharedPtr row_binlog_rowset; - DCHECK_LE(attach_rowsets.size(), 1); - if (!attach_rowsets.empty()) { - row_binlog_rowset = attach_rowsets[0]; - } - result = tablet->add_inc_rowset(rowset, row_binlog_rowset); + auto add_inc_rowset = [&](const TabletSharedPtr& dst_tablet, + const RowsetSharedPtr& dst_rowset) -> Status { + int64_t start_time = MonotonicMicros(); + auto st = dst_tablet->add_inc_rowset(dst_rowset); + stats.add_inc_rowset_us += MonotonicMicros() - start_time; + if (!st.ok() && !st.is()) { + LOG(WARNING) << "fail to add visible rowset to tablet. rowset_id=" + << dst_rowset->rowset_id() << ", tablet_id=" << dst_tablet->tablet_id() + << ", txn_id=" << transaction_id << ", res=" << st; + if (engine_publish_version_task) { + engine_publish_version_task->add_error_tablet_id(dst_tablet->tablet_id()); + } + } + return st; + }; + + result = add_inc_rowset(tablet, rowset); DBUG_EXECUTE_IF("EnginePublishVersionTask.handle.after_add_inc_rowset_rowsets_block", DBUG_BLOCK); - stats.add_inc_rowset_us = MonotonicMicros() - start_time; if (!result.ok() && !result.is()) { - LOG(WARNING) << "fail to add visible rowset to tablet. rowset_id=" << rowset->rowset_id() - << ", tablet_id=" << tablet->tablet_id() << ", txn_id=" << transaction_id - << ", res=" << result; - if (engine_publish_version_task) { - engine_publish_version_task->add_error_tablet_id(tablet->tablet_id()); - } return result; } + // the row binlog rowset is added to its own independent binlog tablet version chain. + if (extend_tablet_txn_info_lifetime != nullptr && + extend_tablet_txn_info_lifetime->attach_row_binlog.rowset != nullptr) { + const auto& attach_row_binlog = extend_tablet_txn_info_lifetime->attach_row_binlog; + auto binlog_tablet = std::static_pointer_cast(attach_row_binlog.tablet); + result = add_inc_rowset(binlog_tablet, attach_row_binlog.rowset); + if (!result.ok() && !result.is()) { + return result; + } + } + return result; } +// Acquire the migration read lock of a tablet with a 5s timeout. +static Status try_lock_migration(const TabletSharedPtr& tablet, int64_t transaction_id, + std::shared_lock& lock) { + lock = std::shared_lock(tablet->get_migration_lock(), + std::chrono::seconds(5)); + if (!lock.owns_lock()) { + auto st = Status::Error("got migration_rlock failed, tablet_id={}", + tablet->tablet_id()); + LOG(WARNING) << "failed to publish version. tablet_id=" << tablet->tablet_id() + << ", txn_id=" << transaction_id << ", res=" << st; + return st; + } + return Status::OK(); +} + void TabletPublishTxnTask::handle() { - std::shared_lock migration_rlock(_tablet->get_migration_lock(), std::chrono::seconds(5)); SCOPED_ATTACH_TASK(_mem_tracker); - if (!migration_rlock.owns_lock()) { - _result = Status::Error("got migration_rlock failed"); - LOG(WARNING) << "failed to publish version. tablet_id=" << _tablet_info.tablet_id - << ", txn_id=" << _transaction_id << ", res=" << _result; + // the row binlog is published to its own binlog tablet together with the base tablet, acquire + // both tablets' locks in binlog-first order. + auto binlog_tablet = std::static_pointer_cast(_attach_row_binlog.tablet); + std::shared_lock binlog_migration_rlock; + std::shared_lock migration_rlock; + if (binlog_tablet != nullptr) { + if (_result = try_lock_migration(binlog_tablet, _transaction_id, binlog_migration_rlock); + !_result.ok()) { + return; + } + } + if (_result = try_lock_migration(_tablet, _transaction_id, migration_rlock); !_result.ok()) { return; } + + std::unique_lock binlog_rowset_update_lock; + if (binlog_tablet != nullptr) { + binlog_rowset_update_lock = + std::unique_lock(binlog_tablet->get_rowset_update_lock()); + } std::unique_lock rowset_update_lock(_tablet->get_rowset_update_lock(), std::defer_lock); if (_tablet->enable_unique_key_merge_on_write()) { @@ -497,7 +538,7 @@ void TabletPublishTxnTask::handle() { } _stats.schedule_time_us = MonotonicMicros() - _stats.submit_time_us; _result = publish_version_and_add_rowset(_engine, _partition_id, _tablet, _rowset, - _attach_rowsets, _transaction_id, _version, + _transaction_id, _version, _engine_publish_version_task, _stats, _commit_tso); if (!_result.ok()) { @@ -518,32 +559,46 @@ void TabletPublishTxnTask::handle() { } void AsyncTabletPublishTask::handle() { - std::shared_lock migration_rlock(_tablet->get_migration_lock(), std::chrono::seconds(5)); SCOPED_ATTACH_TASK(_mem_tracker); - if (!migration_rlock.owns_lock()) { - LOG(WARNING) << "failed to publish version. tablet_id=" << _tablet->tablet_id() - << ", txn_id=" << _transaction_id << ", got migration_rlock failed"; - return; - } - std::lock_guard wrlock(_tablet->get_rowset_update_lock()); - _stats.schedule_time_us = MonotonicMicros() - _stats.submit_time_us; std::map tablet_related_rs; - std::map> tablet_related_attach_rowsets; - _engine.txn_manager()->get_txn_related_tablets( - _transaction_id, _partition_id, &tablet_related_rs, &tablet_related_attach_rowsets); + std::map> tablet_related_txn_infos; + _engine.txn_manager()->get_txn_related_tablets(_transaction_id, _partition_id, + &tablet_related_rs, &tablet_related_txn_infos); auto iter = tablet_related_rs.find(TabletInfo(_tablet->tablet_id(), _tablet->tablet_uid())); if (iter == tablet_related_rs.end()) { return; } - auto attach_rowsets_it = tablet_related_attach_rowsets.find( - TabletInfo(_tablet->tablet_id(), _tablet->tablet_uid())); - DCHECK(attach_rowsets_it != tablet_related_attach_rowsets.end()); + auto txn_info_it = + tablet_related_txn_infos.find(TabletInfo(_tablet->tablet_id(), _tablet->tablet_uid())); + DCHECK(txn_info_it != tablet_related_txn_infos.end()); RowsetSharedPtr rowset = iter->second; Version version(_version, _version); - auto publish_status = publish_version_and_add_rowset(_engine, _partition_id, _tablet, rowset, - attach_rowsets_it->second, _transaction_id, - version, nullptr, _stats, _commit_tso); + // the row binlog is published to its own binlog tablet together with the base tablet, acquire + // both tablets' locks in binlog-first order. + auto binlog_tablet = + std::static_pointer_cast(txn_info_it->second->attach_row_binlog.tablet); + std::shared_lock binlog_migration_rlock; + std::shared_lock migration_rlock; + if (binlog_tablet != nullptr) { + if (!try_lock_migration(binlog_tablet, _transaction_id, binlog_migration_rlock).ok()) { + return; + } + } + if (!try_lock_migration(_tablet, _transaction_id, migration_rlock).ok()) { + return; + } + std::unique_lock binlog_rowset_update_lock; + if (binlog_tablet != nullptr) { + binlog_rowset_update_lock = + std::unique_lock(binlog_tablet->get_rowset_update_lock()); + } + std::lock_guard wrlock(_tablet->get_rowset_update_lock()); + _stats.schedule_time_us = MonotonicMicros() - _stats.submit_time_us; + + auto publish_status = + publish_version_and_add_rowset(_engine, _partition_id, _tablet, rowset, _transaction_id, + version, nullptr, _stats, _commit_tso); if (!publish_status.ok()) { return; diff --git a/be/src/storage/task/engine_publish_version_task.h b/be/src/storage/task/engine_publish_version_task.h index e2edb36db17fdc..a83d7196291208 100644 --- a/be/src/storage/task/engine_publish_version_task.h +++ b/be/src/storage/task/engine_publish_version_task.h @@ -32,6 +32,7 @@ #include "storage/rowset/rowset_fwd.h" #include "storage/tablet/tablet_fwd.h" #include "storage/task/engine_task.h" +#include "storage/txn/txn_manager.h" #include "util/time.h" namespace doris { @@ -65,7 +66,7 @@ class TabletPublishTxnTask { public: TabletPublishTxnTask(StorageEngine& engine, EnginePublishVersionTask* engine_task, TabletSharedPtr tablet, RowsetSharedPtr rowset, - std::vector attach_rowsets, int64_t partition_id, + const RowBinlogTxnInfo& attach_row_binlog, int64_t partition_id, int64_t transaction_id, Version version, const TabletInfo& tablet_info, int64_t commit_tso); ~TabletPublishTxnTask(); @@ -79,7 +80,8 @@ class TabletPublishTxnTask { TabletSharedPtr _tablet; RowsetSharedPtr _rowset; - std::vector _attach_rowsets; + // the row binlog published together with the base tablet. + RowBinlogTxnInfo _attach_row_binlog; int64_t _partition_id; int64_t _transaction_id; Version _version; diff --git a/be/src/storage/txn/txn_manager.cpp b/be/src/storage/txn/txn_manager.cpp index 21f8ebb4a4e013..c5ad0fb8c9571d 100644 --- a/be/src/storage/txn/txn_manager.cpp +++ b/be/src/storage/txn/txn_manager.cpp @@ -194,10 +194,10 @@ Status TxnManager::commit_txn(TPartitionId partition_id, const Tablet& tablet, const RowsetSharedPtr& rowset_ptr, PendingRowsetGuard guard, bool is_recovery, std::shared_ptr partial_update_info, - std::vector* attach_rowsets) { + const RowBinlogTxnInfo& attach_row_binlog) { return commit_txn(tablet.data_dir()->get_meta(), partition_id, transaction_id, tablet.tablet_id(), tablet.tablet_uid(), load_id, rowset_ptr, - std::move(guard), is_recovery, partial_update_info, attach_rowsets); + std::move(guard), is_recovery, partial_update_info, attach_row_binlog); } Status TxnManager::publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet, @@ -285,7 +285,7 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, const RowsetSharedPtr& rowset_ptr, PendingRowsetGuard guard, bool is_recovery, std::shared_ptr partial_update_info, - std::vector* attach_rowsets) { + const RowBinlogTxnInfo& attach_row_binlog) { if (partition_id < 1 || transaction_id < 1 || tablet_id < 1) { LOG(WARNING) << "invalid commit req " << " partition_id=" << partition_id << " transaction_id=" << transaction_id @@ -380,22 +380,15 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, // it is under a single txn lock if (!is_recovery) { std::optional binlog_format; - std::map attach_rowset_map; - if (attach_rowsets != nullptr) { - for (const auto& rs : *attach_rowsets) { - if (rs == nullptr) { - continue; - } - attach_rowset_map.emplace(rs->rowset_id(), rs->rowset_meta()->get_rowset_pb()); - } - if (!attach_rowset_map.empty()) { - binlog_format = BinlogFormatPB::ROW; - } + std::optional attach_row_binlog_rowset_meta; + if (attach_row_binlog.rowset != nullptr) { + attach_row_binlog_rowset_meta = + attach_row_binlog.rowset->rowset_meta()->get_rowset_pb(); + binlog_format = BinlogFormatPB::ROW; } - Status save_status = - RowsetMetaManager::save(meta, tablet_uid, rowset_ptr->rowset_id(), - rowset_ptr->rowset_meta()->get_rowset_pb(), binlog_format, - attach_rowset_map.empty() ? nullptr : &attach_rowset_map); + Status save_status = RowsetMetaManager::save(meta, tablet_uid, rowset_ptr->rowset_id(), + rowset_ptr->rowset_meta()->get_rowset_pb(), + binlog_format, attach_row_binlog_rowset_meta); DBUG_EXECUTE_IF("TxnManager.RowsetMetaManager.save_wait", { if (auto wait = dp->param("duration", 0); wait > 0) { LOG_WARNING("TxnManager.RowsetMetaManager.save_wait") @@ -444,8 +437,19 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, { std::lock_guard wrlock(_get_txn_map_lock(transaction_id)); auto load_info = std::make_shared(load_id, rowset_ptr); - if (attach_rowsets != nullptr) { - load_info->attach_rowsets = *attach_rowsets; + load_info->attach_row_binlog = attach_row_binlog; + // resolve the independent binlog tablet in advance for the later publish phase. + if (load_info->attach_row_binlog.rowset != nullptr && + load_info->attach_row_binlog.tablet == nullptr) { + int64_t binlog_tablet_id = + load_info->attach_row_binlog.rowset->rowset_meta()->tablet_id(); + load_info->attach_row_binlog.tablet = + _engine.tablet_manager()->get_tablet(binlog_tablet_id); + if (load_info->attach_row_binlog.tablet == nullptr) { + return Status::Error( + "binlog tablet not found when commit txn, binlog_tablet_id={}, txn_id={}", + binlog_tablet_id, transaction_id); + } } load_info->pending_rs_guard = std::move(guard); if (is_recovery) { @@ -463,12 +467,13 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, } } - // For binlog txn (attached rowsets exist), binlog_delvec is only needed for publish - // phase to copy delete bitmap deltas. - if (attach_rowsets != nullptr && !attach_rowsets->empty()) { + // For binlog txn, the binlog delete bitmap is only needed in the publish phase to + // copy delete bitmap deltas onto the independent binlog tablet. + if (load_info->attach_row_binlog.rowset != nullptr) { TabletSharedPtr t = _engine.tablet_manager()->get_tablet(tablet_id, tablet_uid); if (t != nullptr && t->enable_unique_key_merge_on_write()) { - load_info->binlog_delvec.reset(new DeleteBitmap(t->tablet_id())); + load_info->attach_row_binlog.delete_bitmap.reset( + new DeleteBitmap(load_info->attach_row_binlog.tablet->tablet_id())); } } load_info->commit(); @@ -553,12 +558,9 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, // it maybe a fatal error rowset->make_visible(version, commit_tso); - // Make all attached rowsets visible together. - for (const auto& rs : tablet_txn_info->attach_rowsets) { - if (rs == nullptr) { - continue; - } - rs->make_visible(version, commit_tso); + // Make the attached binlog rowset visible together. + if (tablet_txn_info->attach_row_binlog.rowset != nullptr) { + tablet_txn_info->attach_row_binlog.rowset->make_visible(version, commit_tso); } DBUG_EXECUTE_IF("TxnManager.publish_txn.random_failed_after_save_rs_meta", { @@ -598,7 +600,19 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, stats->calc_delete_bitmap_time_us = t3 - t2; RETURN_IF_ERROR(TabletMetaManager::save_delete_bitmap( tablet->data_dir(), tablet->tablet_id(), tablet_txn_info->delete_bitmap, - tablet_txn_info->binlog_delvec, version.second)); + version.second)); + if (tablet_txn_info->attach_row_binlog.rowset != nullptr) { + DCHECK(tablet_txn_info->attach_row_binlog.tablet != nullptr); + if (tablet_txn_info->attach_row_binlog.delete_bitmap != nullptr) { + auto binlog_tablet = + std::static_pointer_cast(tablet_txn_info->attach_row_binlog.tablet); + binlog_tablet->merge_delete_bitmap( + *tablet_txn_info->attach_row_binlog.delete_bitmap); + RETURN_IF_ERROR(TabletMetaManager::save_delete_bitmap( + binlog_tablet->data_dir(), binlog_tablet->tablet_id(), + tablet_txn_info->attach_row_binlog.delete_bitmap, version.second)); + } + } stats->save_meta_time_us = MonotonicMicros() - t3; } @@ -616,24 +630,18 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, } } - std::map attach_rowset_map; - if (tablet_txn_info != nullptr) { - for (const auto& rs : tablet_txn_info->attach_rowsets) { - if (rs == nullptr) { - continue; - } - attach_rowset_map.emplace(rs->rowset_id(), rs->rowset_meta()->get_rowset_pb()); - } - if (!attach_rowset_map.empty()) { - binlog_format = BinlogFormatPB::ROW; - } + std::optional attach_row_binlog_rowset_meta; + if (tablet_txn_info->attach_row_binlog.rowset != nullptr) { + attach_row_binlog_rowset_meta = + tablet_txn_info->attach_row_binlog.rowset->rowset_meta()->get_rowset_pb(); + binlog_format = BinlogFormatPB::ROW; } /// Step 4: save meta int64_t t5 = MonotonicMicros(); auto status = RowsetMetaManager::save(meta, tablet_uid, rowset->rowset_id(), rowset->rowset_meta()->get_rowset_pb(), binlog_format, - attach_rowset_map.empty() ? nullptr : &attach_rowset_map); + attach_row_binlog_rowset_meta); stats->save_meta_time_us += MonotonicMicros() - t5; if (!status.ok()) { status.append(fmt::format(", txn id: {}", transaction_id)); @@ -774,10 +782,12 @@ Status TxnManager::delete_txn(OlapMeta* meta, TPartitionId partition_id, rowset->rowset_id().to_string(), rowset->version().to_string(), RowsetStatePB_Name(rowset->rowset_meta_state())); } else { - for (const auto& attach_rowset : load_info->attach_rowsets) { - static_cast(RowsetMetaManager::remove_row_binlog( - meta, tablet_uid, rowset->rowset_id(), attach_rowset->rowset_id())); - _engine.add_unused_rowset(attach_rowset); + const auto& attach_binlog_rowset = load_info->attach_row_binlog.rowset; + if (attach_binlog_rowset != nullptr) { + static_cast(RowsetMetaManager::remove( + meta, attach_binlog_rowset->rowset_meta()->tablet_uid(), + attach_binlog_rowset->rowset_id())); + _engine.add_unused_rowset(attach_binlog_rowset); } static_cast(RowsetMetaManager::remove(meta, tablet_uid, rowset->rowset_id())); #ifndef BE_TEST @@ -788,9 +798,9 @@ Status TxnManager::delete_txn(OlapMeta* meta, TPartitionId partition_id, << ", tablet: " << tablet_info.to_string() << ", rowset: " << (rowset != nullptr ? rowset->rowset_id().to_string() : "0") << ", binlog rowset: " - << (load_info->attach_rowsets.empty() - ? "0" - : load_info->attach_rowsets[0]->rowset_id().to_string()); + << (attach_binlog_rowset != nullptr + ? attach_binlog_rowset->rowset_id().to_string() + : "0"); } } it->second.erase(load_itr); @@ -841,18 +851,20 @@ void TxnManager::force_rollback_tablet_related_txns(OlapMeta* meta, TTabletId ta if (load_itr != it->second.end()) { auto& load_info = load_itr->second; auto& rowset = load_info->rowset; + const auto& attach_binlog_rowset = load_info->attach_row_binlog.rowset; if (rowset != nullptr && meta != nullptr) { LOG(INFO) << " delete transaction from engine " << ", tablet: " << tablet_info.to_string() << ", rowset id: " << rowset->rowset_id(); - // clean attach rowset first - for (const auto& attach_rowset : load_info->attach_rowsets) { - Status status = RowsetMetaManager::remove_row_binlog( - meta, tablet_uid, rowset->rowset_id(), attach_rowset->rowset_id()); - if (!status.ok()) { - if (status.is()) { - continue; - } + // clean the attached binlog rowset first + if (attach_binlog_rowset != nullptr) { + Status status = RowsetMetaManager::remove( + meta, attach_binlog_rowset->rowset_meta()->tablet_uid(), + attach_binlog_rowset->rowset_id()); + if (!status.ok() && !status.is()) { + LOG(WARNING) + << "failed to remove binlog rowset meta, rowset_id=" + << attach_binlog_rowset->rowset_id() << ", status=" << status; } } static_cast( @@ -864,9 +876,9 @@ void TxnManager::force_rollback_tablet_related_txns(OlapMeta* meta, TTabletId ta << ", tablet: " << tablet_info.to_string() << ", rowset: " << (rowset != nullptr ? rowset->rowset_id().to_string() : "0") << ", binlog rowset: " - << (load_info->attach_rowsets.empty() - ? "0" - : load_info->attach_rowsets[0]->rowset_id().to_string()); + << (attach_binlog_rowset != nullptr + ? attach_binlog_rowset->rowset_id().to_string() + : "0"); it->second.erase(load_itr); } if (it->second.empty()) { @@ -890,7 +902,7 @@ void TxnManager::force_rollback_tablet_related_txns(OlapMeta* meta, TTabletId ta void TxnManager::get_txn_related_tablets( const TTransactionId transaction_id, TPartitionId partition_id, std::map* tablet_infos, - std::map>* tablet_attach_rowsets) { + std::map>* tablet_txn_infos) { // get tablets in this transaction pair key(partition_id, transaction_id); std::shared_lock txn_rdlock(_get_txn_map_lock(transaction_id)); @@ -909,8 +921,8 @@ void TxnManager::get_txn_related_tablets( // must not check rowset == null here, because if rowset == null // publish version should failed tablet_infos->emplace(tablet_info, load_info.second->rowset); - if (tablet_attach_rowsets != nullptr) { - tablet_attach_rowsets->emplace(tablet_info, load_info.second->attach_rowsets); + if (tablet_txn_infos != nullptr) { + tablet_txn_infos->emplace(tablet_info, load_info.second); } } } diff --git a/be/src/storage/txn/txn_manager.h b/be/src/storage/txn/txn_manager.h index d7d3404822455a..7ff6d9ade03adb 100644 --- a/be/src/storage/txn/txn_manager.h +++ b/be/src/storage/txn/txn_manager.h @@ -63,6 +63,14 @@ enum class TxnState { }; enum class PublishStatus { INIT = 0, PREPARE = 1, SUCCEED = 2 }; +// The row binlog rowset and its independent binlog tablet, carried through commit and publish. +struct RowBinlogTxnInfo { + RowsetSharedPtr rowset; + BaseTabletSPtr tablet; + // Delete bitmap deltas that should be applied to the independent binlog tablet. + DeleteBitmapPtr delete_bitmap; +}; + struct TxnPublishInfo { int64_t publish_version {-1}; int64_t base_compaction_cnt {-1}; @@ -73,14 +81,11 @@ struct TxnPublishInfo { struct TabletTxnInfo { PUniqueId load_id; RowsetSharedPtr rowset; - // The list of rowsets committed along with the transaction rowset - // currently contains only the binlog rowset. - std::vector attach_rowsets; + // the row binlog committed along with this txn. + RowBinlogTxnInfo attach_row_binlog; PendingRowsetGuard pending_rs_guard; bool unique_key_merge_on_write {false}; DeleteBitmapPtr delete_bitmap; - // copy delete_bitmap of data rowset to binlog - DeleteBitmapPtr binlog_delvec; // records rowsets calc in commit txn RowsetIdUnorderedSet rowset_ids; int64_t creation_time; @@ -174,7 +179,7 @@ class TxnManager { TTransactionId transaction_id, const PUniqueId& load_id, const RowsetSharedPtr& rowset_ptr, PendingRowsetGuard guard, bool is_recovery, std::shared_ptr partial_update_info = nullptr, - std::vector* attach_rowsets = nullptr); + const RowBinlogTxnInfo& attach_row_binlog = {}); Status publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet, TTransactionId transaction_id, const Version& version, @@ -193,7 +198,7 @@ class TxnManager { TTabletId tablet_id, TabletUid tablet_uid, const PUniqueId& load_id, const RowsetSharedPtr& rowset_ptr, PendingRowsetGuard guard, bool is_recovery, std::shared_ptr partial_update_info = nullptr, - std::vector* attach_rowsets = nullptr); + const RowBinlogTxnInfo& attach_row_binlog = {}); // remove a txn from txn manager // not persist rowset meta because @@ -223,7 +228,7 @@ class TxnManager { void get_txn_related_tablets( const TTransactionId transaction_id, TPartitionId partition_ids, std::map* tablet_infos, - std::map>* tablet_attach_rowsets = nullptr); + std::map>* tablet_txn_infos = nullptr); void get_all_related_tablets(std::set* tablet_infos); diff --git a/be/test/cloud/cloud_compaction_test.cpp b/be/test/cloud/cloud_compaction_test.cpp index f7916ab511ef55..ec255b02c9c63e 100644 --- a/be/test/cloud/cloud_compaction_test.cpp +++ b/be/test/cloud/cloud_compaction_test.cpp @@ -201,6 +201,37 @@ TEST_F(CloudCompactionTest, failure_cumu_compaction_tablet_sleep_test) { ASSERT_EQ(tablets.size(), 0); } +TEST_F(CloudCompactionTest, binlog_compaction_max_score_ignores_normal_tablets) { + auto filter_out = [](CloudTablet* t) { return !t->is_row_binlog_tablet(); }; + CloudTabletMgr mgr(_engine); + + auto normal_meta = std::make_shared(*_tablet_meta); + normal_meta->set_is_row_binlog_tablet(false); + CloudTabletSPtr normal_tablet = std::make_shared(_engine, normal_meta); + normal_tablet->tablet_meta()->_tablet_id = 10001; + normal_tablet->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false); + normal_tablet->_approximate_cumu_num_deltas = 10; + mgr.put_tablet_for_UT(normal_tablet); + + auto binlog_meta = std::make_shared(*_tablet_meta); + binlog_meta->set_is_row_binlog_tablet(true); + CloudTabletSPtr binlog_tablet = std::make_shared(_engine, binlog_meta); + binlog_tablet->tablet_meta()->_tablet_id = 10002; + binlog_tablet->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false); + binlog_tablet->_approximate_cumu_num_deltas = 7; + mgr.put_tablet_for_UT(binlog_tablet); + + int64_t max_score = 0; + std::vector> tablets {}; + Status st = mgr.get_topn_tablets_to_compact(1, CompactionType::CUMU_BINLOG_COMPACTION, + filter_out, &tablets, &max_score); + + ASSERT_EQ(st, Status::OK()); + ASSERT_EQ(tablets.size(), 1); + EXPECT_EQ(tablets.front()->tablet_id(), binlog_tablet->tablet_id()); + EXPECT_EQ(max_score, 7); +} + static RowsetSharedPtr create_rowset(Version version, int num_segments, bool overlapping, int data_size) { auto rs_meta = std::make_shared(); diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp index 08799c900d2cf2..37d68958df9708 100644 --- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp +++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp @@ -23,6 +23,9 @@ #include #include +#include + +#include "cloud/cloud_cumulative_compaction_binlog_policy.h" #include "cloud/cloud_storage_engine.h" #include "cloud/config.h" #include "common/config.h" @@ -32,6 +35,7 @@ #include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" #include "storage/tablet/tablet_meta.h" +#include "util/time.h" #include "util/uid_util.h" namespace doris { @@ -145,6 +149,36 @@ static int64_t total_disk_size(const std::vector& rowsets) { return total_size; } +class TestCloudBinlogCumulativeCompactionPolicy : public testing::Test { +public: + TestCloudBinlogCumulativeCompactionPolicy() : _engine(CloudStorageEngine(EngineOptions {})) {} + + void SetUp() override { + config::binlog_compaction_file_count_threshold = 3; + config::binlog_compaction_time_threshold_seconds = INT32_MAX; + + _tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F)); + _tablet_meta->set_is_row_binlog_tablet(true); + _tablet_meta->set_compaction_policy(std::string(CUMULATIVE_BINLOG_POLICY)); + } + + RowsetSharedPtr create_binlog_rowset(Version version, int num_segments, bool overlapping, + int data_size, int8_t compaction_level) { + auto rowset = create_rowset(version, num_segments, overlapping, data_size); + rowset->rowset_meta()->set_data_disk_size(data_size); + rowset->rowset_meta()->set_compaction_level(compaction_level); + rowset->rowset_meta()->set_newest_write_timestamp(UnixSeconds()); + rowset->rowset_meta()->mark_row_binlog(); + return rowset; + } + +protected: + CloudStorageEngine _engine; + TabletMetaSharedPtr _tablet_meta; +}; + TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, new_cumulative_point) { std::vector rs_metas; init_rs_meta_small_base(&rs_metas); @@ -267,6 +301,113 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, EXPECT_EQ(0, compaction_score); } +TEST_F(TestCloudBinlogCumulativeCompactionPolicy, pick_level_from_candidate_rowsets) { + CloudTablet tablet(_engine, _tablet_meta); + tablet.set_last_cumu_compaction_success_time(UnixMillis()); + + std::vector candidate_rowsets { + create_binlog_rowset(Version(2, 2), 1, false, 1, 0), + create_binlog_rowset(Version(3, 3), 1, false, 1, 0), + create_binlog_rowset(Version(4, 4), 4, true, 1, 1), + create_binlog_rowset(Version(5, 5), 1, false, 1, 1)}; + + CloudBinlogCumulativeCompactionPolicy policy; + std::vector input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + int picked = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 1, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(2, picked); + EXPECT_EQ(5, compaction_score); + ASSERT_EQ(2, input_rowsets.size()); + EXPECT_EQ(4, input_rowsets.front()->start_version()); + EXPECT_EQ(5, input_rowsets.back()->end_version()); + + tablet.set_cumulative_layer_point(2); + input_rowsets.clear(); + compaction_score = 0; + int8_t max_level = BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1; + candidate_rowsets = {create_binlog_rowset(Version(0, 0), 1, false, 1, max_level), + create_binlog_rowset(Version(1, 1), 1, false, 1, max_level), + create_binlog_rowset(Version(2, 2), 2, true, 1, max_level), + create_binlog_rowset(Version(3, 3), 1, false, 1, max_level)}; + picked = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 1, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(2, picked); + EXPECT_EQ(3, compaction_score); + ASSERT_EQ(2, input_rowsets.size()); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(3, input_rowsets.back()->end_version()); +} + +TEST_F(TestCloudBinlogCumulativeCompactionPolicy, filter_new_visible_rowsets) { + CloudTablet tablet(_engine, _tablet_meta); + tablet.set_last_cumu_compaction_success_time(UnixMillis()); + config::binlog_compaction_wait_timesec_after_visible = 600; + + int8_t level = 0; + std::vector candidate_rowsets { + create_binlog_rowset(Version(2, 2), 1, false, 1, level), + create_binlog_rowset(Version(3, 3), 2, true, 1, level), + create_binlog_rowset(Version(4, 4), 1, false, 1, level)}; + candidate_rowsets[0]->rowset_meta()->set_newest_write_timestamp( + UnixSeconds() - config::binlog_compaction_wait_timesec_after_visible - 1); + candidate_rowsets[1]->rowset_meta()->set_newest_write_timestamp(UnixSeconds()); + candidate_rowsets[2]->rowset_meta()->set_newest_write_timestamp(UnixSeconds()); + + CloudBinlogCumulativeCompactionPolicy policy; + std::vector input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + int picked = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 1, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(0, picked); + EXPECT_EQ(0, compaction_score); + EXPECT_TRUE(input_rowsets.empty()); + + candidate_rowsets[1]->rowset_meta()->set_newest_write_timestamp( + UnixSeconds() - config::binlog_compaction_wait_timesec_after_visible - 1); + picked = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 1, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(2, picked); + EXPECT_EQ(3, compaction_score); + ASSERT_EQ(2, input_rowsets.size()); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(3, input_rowsets.back()->end_version()); + + candidate_rowsets.emplace_back(create_binlog_rowset(Version(5, 5), 1, false, 1, level)); + input_rowsets.clear(); + compaction_score = 0; + picked = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 1, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(4, picked); + EXPECT_EQ(5, compaction_score); + ASSERT_EQ(4, input_rowsets.size()); + EXPECT_EQ(5, input_rowsets.back()->end_version()); +} + +TEST_F(TestCloudBinlogCumulativeCompactionPolicy, new_cumulative_point) { + CloudTablet tablet(_engine, _tablet_meta); + CloudBinlogCumulativeCompactionPolicy policy; + Version last_delete_version(1, 1); + int8_t max_level = BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1; + int64_t goal_size = config::binlog_compaction_goal_size_mbytes * 1024 * 1024; + + auto output_rowset = create_binlog_rowset(Version(3, 5), 1, false, goal_size, max_level); + EXPECT_EQ(policy.new_cumulative_point(&tablet, output_rowset, last_delete_version, -1), 6); + + auto small_output_rowset = create_binlog_rowset(Version(6, 7), 1, false, 1, max_level); + EXPECT_EQ(policy.new_cumulative_point(&tablet, small_output_rowset, last_delete_version, 6), 6); + + auto empty_output_rowset = create_binlog_rowset(Version(8, 9), 0, false, goal_size, max_level); + EXPECT_EQ(policy.new_cumulative_point(&tablet, empty_output_rowset, last_delete_version, 6), 6); +} + // Test case: Empty rowset compaction with skip_trim TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_empty_rowset_compaction) { // Save original config values diff --git a/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp new file mode 100644 index 00000000000000..93d11fdbf80d04 --- /dev/null +++ b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "cloud/cloud_rowset_builder.h" +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_tablet_mgr.h" +#include "common/cast_set.h" +#include "common/config.h" +#include "core/block/block.h" +#include "core/field.h" +#include "exec/sink/autoinc_buffer.h" +#include "io/fs/local_file_system.h" +#include "io/fs/remote_file_system.h" +#include "runtime/descriptor_helper.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_profile.h" +#include "storage/binlog.h" +#include "storage/rowset/group_rowset_writer.h" +#include "storage/rowset/rowset_factory.h" +#include "storage/rowset/rowset_writer_context.h" +#include "storage/storage_policy.h" +#include "storage/tablet/tablet_meta.h" +#include "storage/tablet/tablet_schema.h" +#include "testutil/creators.h" +#include "util/uid_util.h" + +namespace doris { +namespace { + +constexpr uint32_t MAX_PATH_LEN = 1024; +constexpr std::string_view kStorageRootDir = "/ut_dir/cloud_group_rowset_builder_writer_test"; +constexpr int64_t kDataTabletId = 10010; +constexpr int64_t kRowBinlogTabletId = 10011; +constexpr int64_t kDataIndexId = 10001; +constexpr int64_t kRowBinlogIndexId = 10002; + +class LocalRemoteFileSystem final : public io::RemoteFileSystem { +public: + explicit LocalRemoteFileSystem(std::string root_path) + : RemoteFileSystem(std::move(root_path), "cloud_group_rowset_test_fs", + io::FileSystemType::BROKER) {} + +private: + Status create_file_impl(const io::Path& file, io::FileWriterPtr* writer, + const io::FileWriterOptions* opts) override { + return io::global_local_filesystem()->create_file(file, writer, opts); + } + + Status create_directory_impl(const io::Path& dir, bool failed_if_exists) override { + return io::global_local_filesystem()->create_directory(dir, failed_if_exists); + } + + Status delete_file_impl(const io::Path& file) override { + return io::global_local_filesystem()->delete_file(file); + } + + Status batch_delete_impl(const std::vector& files) override { + return io::global_local_filesystem()->batch_delete(files); + } + + Status delete_directory_impl(const io::Path& dir) override { + return io::global_local_filesystem()->delete_directory(dir); + } + + Status exists_impl(const io::Path& path, bool* res) const override { + return io::global_local_filesystem()->exists(path, res); + } + + Status file_size_impl(const io::Path& file, int64_t* file_size) const override { + return io::global_local_filesystem()->file_size(file, file_size); + } + + Status list_impl(const io::Path& dir, bool only_file, std::vector* files, + bool* exists) override { + return io::global_local_filesystem()->list(dir, only_file, files, exists); + } + + Status rename_impl(const io::Path& orig_name, const io::Path& new_name) override { + return io::global_local_filesystem()->rename(orig_name, new_name); + } + + Status upload_impl(const io::Path& local_file, const io::Path& remote_file) override { + return io::global_local_filesystem()->link_file(local_file, remote_file); + } + + Status batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) override { + DCHECK_EQ(local_files.size(), remote_files.size()); + for (size_t i = 0; i < local_files.size(); ++i) { + RETURN_IF_ERROR(upload_impl(local_files[i], remote_files[i])); + } + return Status::OK(); + } + + Status download_impl(const io::Path& remote_file, const io::Path& local_file) override { + return io::global_local_filesystem()->link_file(remote_file, local_file); + } + + Status open_file_internal(const io::Path& file, io::FileReaderSPtr* reader, + const io::FileReaderOptions& opts) override { + return io::global_local_filesystem()->open_file(file, reader, &opts); + } +}; + +TabletMetaSharedPtr create_tablet_meta(const TCreateTabletReq& request, bool is_row_binlog_tablet) { + std::unordered_map col_idx_to_unique_id; + for (uint32_t col_idx = 0; col_idx < request.tablet_schema.columns.size(); ++col_idx) { + col_idx_to_unique_id[col_idx] = col_idx; + } + auto tablet_meta = TabletMeta::create(request, TabletUid::gen_uid(), 0, + cast_set(request.tablet_schema.columns.size()), + col_idx_to_unique_id); + tablet_meta->set_is_row_binlog_tablet(is_row_binlog_tablet); + return tablet_meta; +} + +void add_initial_rowset(const CloudTabletSPtr& tablet, int64_t version) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_rowset_state(VISIBLE); + rs_meta->set_tablet_id(tablet->tablet_id()); + rs_meta->set_tablet_schema_hash(tablet->schema_hash()); + rs_meta->set_version(Version(0, version)); + rs_meta->set_segments_overlap(OVERLAP_UNKNOWN); + rs_meta->set_tablet_schema(tablet->tablet_schema()); + RowsetSharedPtr rowset; + ASSERT_TRUE(RowsetFactory::create_rowset(tablet->tablet_schema(), "", rs_meta, &rowset).ok()); + std::unique_lock meta_lock(tablet->get_header_lock()); + tablet->add_rowsets({rowset}, false, meta_lock); + tablet->set_cumulative_layer_point(version + 1); +} + +} // namespace + +class CloudGroupRowsetBuilderWriterTest : public testing::Test { +protected: + void SetUp() override { + config::cloud_mow_sync_rowsets_when_load_txn_begin = false; + + char buffer[MAX_PATH_LEN]; + ASSERT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); + _storage_root_path = std::string(buffer) + std::string(kStorageRootDir); + ASSERT_TRUE(io::global_local_filesystem()->delete_directory(_storage_root_path).ok()); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(_storage_root_path).ok()); + ASSERT_TRUE( + io::global_local_filesystem()->create_directory(_storage_root_path + "/data").ok()); + + _engine = std::make_unique(EngineOptions {}); + _engine->init_calc_delete_bitmap_executor_for_UT(); + _engine->set_latest_fs(std::make_shared(_storage_root_path)); + + _request = testutil::create_tablet_request( + kDataTabletId, 270068390, 10001, 1, TKeysType::UNIQUE_KEYS, + {{"k1", TPrimitiveType::INT, true}, {"v1", TPrimitiveType::INT, false}}); + _request.__set_enable_unique_key_merge_on_write(true); + testutil::enable_row_binlog(&_request); + + _row_binlog_request = _request; + _row_binlog_request.tablet_id = kRowBinlogTabletId; + _row_binlog_request.tablet_schema = testutil::create_row_binlog_tablet_schema( + _request.tablet_schema, _request.tablet_schema.schema_hash + 1); + _row_binlog_request.__set_is_row_binlog_tablet(true); + + auto data_meta = create_tablet_meta(_request, false); + auto row_binlog_meta = create_tablet_meta(_row_binlog_request, true); + _tablet = std::make_shared(*_engine, data_meta); + _row_binlog_tablet = std::make_shared(*_engine, row_binlog_meta); + add_initial_rowset(_tablet, _request.version); + add_initial_rowset(_row_binlog_tablet, _row_binlog_request.version); + _engine->tablet_mgr().put_tablet_for_UT(_tablet); + _engine->tablet_mgr().put_tablet_for_UT(_row_binlog_tablet); + create_remote_tablet_dir(kDataTabletId); + create_remote_tablet_dir(kRowBinlogTabletId); + } + + void TearDown() override { + _tablet.reset(); + _row_binlog_tablet.reset(); + _engine.reset(); + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_storage_root_path).ok()); + config::cloud_mow_sync_rowsets_when_load_txn_begin = true; + } + + void create_remote_tablet_dir(int64_t tablet_id) { + ASSERT_TRUE( + io::global_local_filesystem() + ->create_directory(fmt::format("{}/data/{}", _storage_root_path, tablet_id)) + .ok()); + } + + std::shared_ptr create_schema_param() { + TDescriptorTable tdesc_tbl = testutil::create_descriptor_table( + {{TYPE_INT, "k1", false}, {TYPE_INT, "v1", false}}); + return testutil::create_table_schema_param( + tdesc_tbl, kDataIndexId, _request.tablet_schema.schema_hash, + _request.tablet_schema.columns, kRowBinlogIndexId, + _row_binlog_request.tablet_schema.schema_hash, + &_row_binlog_request.tablet_schema.columns); + } + + void init_write_requests(WriteRequest* data_req, WriteRequest* row_binlog_req, + WriteRequest* group_req) { + PUniqueId load_id; + load_id.set_hi(0); + load_id.set_lo(0); + + *data_req = {}; + data_req->tablet_id = _request.tablet_id; + data_req->schema_hash = _request.tablet_schema.schema_hash; + data_req->txn_id = 20010; + data_req->partition_id = _request.partition_id; + data_req->index_id = kDataIndexId; + data_req->load_id = load_id; + data_req->table_schema_param = create_schema_param(); + data_req->write_req_type = WriteRequestType::DATA; + + *row_binlog_req = *data_req; + row_binlog_req->tablet_id = _row_binlog_request.tablet_id; + row_binlog_req->index_id = kRowBinlogIndexId; + row_binlog_req->schema_hash = _row_binlog_request.tablet_schema.schema_hash; + row_binlog_req->write_req_type = WriteRequestType::ROW_BINLOG; + + *group_req = *data_req; + group_req->write_req_type = WriteRequestType::GROUP; + } + + Block create_block(int start_key, int num_rows) const { + Block block = _tablet->tablet_schema()->create_block(); + { + auto columns_guard = block.mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + for (int i = 0; i < num_rows; ++i) { + columns[0]->insert(Field::create_field(start_key + i)); + columns[1]->insert( + Field::create_field((start_key + i) * 10)); + } + } + return block; + } + + Status create_group_writer(std::unique_ptr* group_writer, int num_rows) { + RowsetWriterContext data_context; + data_context.rowset_state = PREPARED; + data_context.segments_overlap = OVERLAPPING; + data_context.tablet_schema = _tablet->tablet_schema(); + data_context.tablet = _tablet; + data_context.tablet_schema_hash = _request.tablet_schema.schema_hash; + data_context.max_rows_per_segment = 1024; + data_context.write_type = DataWriteType::TYPE_DIRECT; + data_context.storage_resource = _engine->get_storage_resource(""); + auto data_writer_res = _tablet->create_rowset_writer(data_context, false); + if (!data_writer_res.has_value()) { + return data_writer_res.error(); + } + + RowsetWriterContext row_binlog_context; + row_binlog_context.rowset_state = PREPARED; + row_binlog_context.segments_overlap = NONOVERLAPPING; + row_binlog_context.tablet_schema = _row_binlog_tablet->tablet_schema(); + row_binlog_context.tablet = _row_binlog_tablet; + row_binlog_context.tablet_schema_hash = _row_binlog_request.tablet_schema.schema_hash; + row_binlog_context.max_rows_per_segment = 1024; + row_binlog_context.write_type = DataWriteType::TYPE_DIRECT; + row_binlog_context.storage_resource = _engine->get_storage_resource(""); + row_binlog_context.write_binlog_opt().enable = true; + auto& cfg = row_binlog_context.write_binlog_opt().write_binlog_config(); + cfg.source.tablet_schema = _tablet->tablet_schema(); + cfg.source.base_tablet = _tablet; + cfg.source.source_write_type = DataWriteType::TYPE_DIRECT; + auto lsn_buffer = AutoIncIDBuffer::create_shared(1, 1, kBinlogLsnAutoIncId); + lsn_buffer->append_range_for_test(1000, num_rows); + auto lsn_ids = std::make_shared>(); + RETURN_IF_ERROR(allocate_binlog_lsn(lsn_buffer, num_rows, *lsn_ids)); + cfg.insert_seg_lsn(0, lsn_ids); + + auto row_binlog_writer_res = + _row_binlog_tablet->create_rowset_writer(row_binlog_context, false); + if (!row_binlog_writer_res.has_value()) { + return row_binlog_writer_res.error(); + } + + *group_writer = std::make_unique(); + (*group_writer) + ->set_data_writer( + std::shared_ptr(std::move(data_writer_res.value()))); + (*group_writer) + ->set_row_binlog_writer( + std::shared_ptr(std::move(row_binlog_writer_res.value()))); + return Status::OK(); + } + + void assert_rowset_meta(const RowsetSharedPtr& data_rowset, + const RowsetSharedPtr& row_binlog_rowset) { + ASSERT_FALSE(data_rowset->rowset_meta()->is_row_binlog()); + ASSERT_EQ(data_rowset->rowset_meta()->tablet_id(), _tablet->tablet_id()); + ASSERT_EQ(data_rowset->rowset_meta()->tablet_schema_hash(), + _request.tablet_schema.schema_hash); + + ASSERT_TRUE(row_binlog_rowset->rowset_meta()->is_row_binlog()); + ASSERT_EQ(row_binlog_rowset->rowset_meta()->tablet_id(), _row_binlog_tablet->tablet_id()); + ASSERT_EQ(row_binlog_rowset->rowset_meta()->tablet_schema_hash(), + _row_binlog_request.tablet_schema.schema_hash); + ASSERT_GE(row_binlog_rowset->rowset_meta()->tablet_schema()->field_index(BINLOG_LSN_COL), + 0); + } + + std::unique_ptr _engine; + CloudTabletSPtr _tablet; + CloudTabletSPtr _row_binlog_tablet; + TCreateTabletReq _request; + TCreateTabletReq _row_binlog_request; + std::string _storage_root_path; +}; + +TEST_F(CloudGroupRowsetBuilderWriterTest, builderBuildsRowBinlogMeta) { + WriteRequest data_req; + WriteRequest row_binlog_req; + WriteRequest group_req; + init_write_requests(&data_req, &row_binlog_req, &group_req); + + RuntimeProfile profile("CloudGroupRowsetBuilderWriterTest"); + CloudGroupRowsetBuilder builder(*_engine, group_req, data_req, row_binlog_req, &profile); + builder.set_skip_writing_rowset_metadata(true); + auto st = builder.init(); + ASSERT_TRUE(st.ok()) << st; + + ASSERT_TRUE(builder.rowset_writer()->flush().ok()); + ASSERT_TRUE(builder.build_rowset().ok()); + + assert_rowset_meta(builder.data_builder()->rowset(), builder.row_binlog_builder()->rowset()); +} + +TEST_F(CloudGroupRowsetBuilderWriterTest, writerBuildsRowBinlogMeta) { + std::unique_ptr group_writer; + auto st = create_group_writer(&group_writer, 2); + ASSERT_TRUE(st.ok()) << st; + + auto block = create_block(100, 2); + st = group_writer->flush_single_block(&block); + ASSERT_TRUE(st.ok()) << st; + + std::vector rowsets; + st = group_writer->build_rowsets(rowsets); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(2, rowsets.size()); + + assert_rowset_meta(rowsets[0], rowsets[1]); +} + +} // namespace doris diff --git a/be/test/olap/rowset/group_rowset_builder_test.cpp b/be/test/olap/rowset/group_rowset_builder_test.cpp index 4bb9c7f98b258a..bd1f3c81fc623a 100644 --- a/be/test/olap/rowset/group_rowset_builder_test.cpp +++ b/be/test/olap/rowset/group_rowset_builder_test.cpp @@ -98,8 +98,17 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) { TabletSharedPtr tablet = engine_ref->tablet_manager()->get_tablet(request.tablet_id); ASSERT_TRUE(tablet != nullptr); - auto st = io::global_local_filesystem()->create_directory(tablet->row_binlog_path()); - ASSERT_TRUE(st.ok()) << st; + + auto row_binlog_request = request; + row_binlog_request.tablet_id = 10011; + auto row_binlog_tablet_schema = testutil::create_row_binlog_tablet_schema( + request.tablet_schema, request.tablet_schema.schema_hash + 1); + row_binlog_request.tablet_schema = row_binlog_tablet_schema; + res = engine_ref->create_tablet(row_binlog_request, profile.get()); + ASSERT_TRUE(res.ok()); + TabletSharedPtr row_binlog_tablet = + engine_ref->tablet_manager()->get_tablet(row_binlog_request.tablet_id); + ASSERT_TRUE(row_binlog_tablet != nullptr); PUniqueId load_id; load_id.set_hi(0); @@ -111,8 +120,8 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) { testutil::create_descriptor_table({{TYPE_INT, "k1", false}, {TYPE_INT, "v1", false}}); auto param = testutil::create_table_schema_param( tdesc_tbl, index_id, request.tablet_schema.schema_hash, request.tablet_schema.columns, - row_binlog_index_id, request.row_binlog_schema.schema_hash, - &request.row_binlog_schema.columns); + row_binlog_index_id, row_binlog_tablet_schema.schema_hash, + &row_binlog_tablet_schema.columns); ASSERT_NE(param, nullptr); WriteRequest data_req; @@ -126,8 +135,9 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) { data_req.write_req_type = WriteRequestType::DATA; WriteRequest row_binlog_req = data_req; + row_binlog_req.tablet_id = row_binlog_request.tablet_id; row_binlog_req.index_id = row_binlog_index_id; - row_binlog_req.schema_hash = request.row_binlog_schema.schema_hash; + row_binlog_req.schema_hash = row_binlog_tablet_schema.schema_hash; row_binlog_req.write_req_type = WriteRequestType::ROW_BINLOG; WriteRequest group_req = data_req; @@ -142,7 +152,7 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) { auto data_meta = builder.txn_rowset_builder()->rowset()->rowset_meta(); ASSERT_TRUE(row_binlog_meta->is_row_binlog()); ASSERT_FALSE(data_meta->is_row_binlog()); - ASSERT_EQ(request.row_binlog_schema.schema_hash, row_binlog_meta->tablet_schema_hash()); + ASSERT_EQ(row_binlog_tablet_schema.schema_hash, row_binlog_meta->tablet_schema_hash()); ASSERT_EQ(request.tablet_schema.schema_hash, data_meta->tablet_schema_hash()); ASSERT_EQ(row_binlog_index_id, row_binlog_meta->index_id()); ASSERT_EQ(index_id, data_meta->index_id()); @@ -152,6 +162,9 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) { res = engine_ref->tablet_manager()->drop_tablet(request.tablet_id, request.replica_id, false); ASSERT_TRUE(res.ok()); + res = engine_ref->tablet_manager()->drop_tablet(row_binlog_request.tablet_id, + row_binlog_request.replica_id, false); + ASSERT_TRUE(res.ok()); } } // namespace doris diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index edf616f323b9e5..76639b257eaf13 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -58,6 +58,9 @@ constexpr std::string_view kStorageRootDir = "/ut_dir/group_rowset_writer_test"; class GroupRowsetWriterTest : public testing::Test { protected: void SetUp() override { + _saved_aggregate_non_mow_key_bounds = config::enable_aggregate_non_mow_key_bounds; + config::enable_aggregate_non_mow_key_bounds = true; + char buffer[MAX_PATH_LEN]; getcwd(buffer, MAX_PATH_LEN); @@ -90,17 +93,21 @@ class GroupRowsetWriterTest : public testing::Test { _request.tablet_schema.columns[4].__set_is_allow_null(true); _request.__set_enable_unique_key_merge_on_write(true); testutil::enable_row_binlog(&_request); - _request.row_binlog_schema.columns.erase(_request.row_binlog_schema.columns.begin() + 5); - _request.row_binlog_schema.columns.erase(_request.row_binlog_schema.columns.begin() + 2); - _request.row_binlog_schema.__set_binlog_tso_idx(4); - _request.row_binlog_schema.__set_binlog_lsn_idx(5); - _request.row_binlog_schema.__set_binlog_op_idx(6); auto profile = std::make_unique("GroupRowsetWriterTest"); ASSERT_TRUE(engine_ptr->create_tablet(_request, profile.get()).ok()); _tablet = engine_ptr->tablet_manager()->get_tablet(_request.tablet_id); ASSERT_TRUE(_tablet != nullptr); - EXPECT_TRUE( - io::global_local_filesystem()->create_directory(_tablet->row_binlog_path()).ok()); + ASSERT_TRUE(_tablet->need_read_delete_bitmap()); + _row_binlog_request = _request; + _row_binlog_request.tablet_id = 10011; + _row_binlog_request.tablet_schema = testutil::create_row_binlog_tablet_schema( + _request.tablet_schema, _request.tablet_schema.schema_hash + 1); + _row_binlog_request.__set_is_row_binlog_tablet(true); + ASSERT_TRUE(engine_ptr->create_tablet(_row_binlog_request, profile.get()).ok()); + _row_binlog_tablet = + engine_ptr->tablet_manager()->get_tablet(_row_binlog_request.tablet_id); + ASSERT_TRUE(_row_binlog_tablet != nullptr); + ASSERT_TRUE(_row_binlog_tablet->need_read_delete_bitmap()); config::enable_debug_points = true; } @@ -109,8 +116,10 @@ class GroupRowsetWriterTest : public testing::Test { DebugPoints::instance()->clear(); config::enable_debug_points = false; _tablet.reset(); + _row_binlog_tablet.reset(); ExecEnv::GetInstance()->set_storage_engine(nullptr); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_storage_root_path).ok()); + config::enable_aggregate_non_mow_key_bounds = _saved_aggregate_non_mow_key_bounds; } Block create_block(int start_key, int num_rows) const { @@ -163,8 +172,8 @@ class GroupRowsetWriterTest : public testing::Test { } RowsetWriterContext row_binlog_context; - row_binlog_context.tablet = _tablet; - row_binlog_context.tablet_schema = _tablet->row_binlog_tablet_schema(); + row_binlog_context.tablet = _row_binlog_tablet; + row_binlog_context.tablet_schema = _row_binlog_tablet->tablet_schema(); row_binlog_context.rowset_state = PREPARED; row_binlog_context.segments_overlap = NONOVERLAPPING; row_binlog_context.max_rows_per_segment = 1024; @@ -180,7 +189,8 @@ class GroupRowsetWriterTest : public testing::Test { auto lsn_ids = std::make_shared>(); RETURN_IF_ERROR(allocate_binlog_lsn(lsn_buffer, num_rows, *lsn_ids)); cfg.insert_seg_lsn(0, lsn_ids); - auto row_binlog_writer_res = _tablet->create_rowset_writer(row_binlog_context, false); + auto row_binlog_writer_res = + _row_binlog_tablet->create_rowset_writer(row_binlog_context, false); if (!row_binlog_writer_res.has_value()) { return row_binlog_writer_res.error(); } @@ -202,8 +212,11 @@ class GroupRowsetWriterTest : public testing::Test { } TabletSharedPtr _tablet; + TabletSharedPtr _row_binlog_tablet; TCreateTabletReq _request; + TCreateTabletReq _row_binlog_request; std::string _storage_root_path; + bool _saved_aggregate_non_mow_key_bounds = false; }; TEST_F(GroupRowsetWriterTest, sub_writer_rollback) { @@ -256,16 +269,29 @@ TEST_F(GroupRowsetWriterTest, success) { const auto data_segment_path = local_segment_path(_tablet->tablet_path(), data_rowset_id.to_string(), 0); - const auto second_segment_path = - local_segment_path(_tablet->row_binlog_path(), rowsets[1]->rowset_id().to_string(), 0); + const auto binlog_segment_path = local_segment_path(_row_binlog_tablet->tablet_path(), + rowsets[1]->rowset_id().to_string(), 0); EXPECT_TRUE(file_exists(data_segment_path)); - EXPECT_TRUE(file_exists(second_segment_path)); + EXPECT_TRUE(file_exists(binlog_segment_path)); + + ASSERT_FALSE(rowsets[0]->rowset_meta()->is_row_binlog()); + ASSERT_EQ(rowsets[0]->rowset_meta()->tablet_id(), _tablet->tablet_id()); + ASSERT_EQ(rowsets[0]->rowset_meta()->tablet_schema_hash(), _request.tablet_schema.schema_hash); + ASSERT_TRUE(rowsets[1]->rowset_meta()->is_row_binlog()); + ASSERT_EQ(rowsets[1]->rowset_meta()->tablet_id(), _row_binlog_tablet->tablet_id()); + ASSERT_EQ(rowsets[1]->rowset_meta()->tablet_schema_hash(), + _row_binlog_request.tablet_schema.schema_hash); + ASSERT_GE(rowsets[1]->rowset_meta()->tablet_schema()->field_index(BINLOG_LSN_COL), 0); + EXPECT_FALSE(rowsets[1]->rowset_meta()->is_segments_key_bounds_aggregated()); + std::vector row_binlog_key_bounds; + rowsets[1]->rowset_meta()->get_segments_key_bounds(&row_binlog_key_bounds); + EXPECT_EQ(row_binlog_key_bounds.size(), rowsets[1]->num_segments()); } TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { - ASSERT_EQ(1, _tablet->row_binlog_tablet_schema()->field_index("__DORIS_TEST_HIDDEN_KEY__")); - ASSERT_EQ(-1, _tablet->row_binlog_tablet_schema()->field_index("__DORIS_TEST_HIDDEN_VALUE__")); - ASSERT_EQ(-1, _tablet->row_binlog_tablet_schema()->field_index(DELETE_SIGN)); + ASSERT_EQ(1, _row_binlog_tablet->tablet_schema()->field_index("__DORIS_TEST_HIDDEN_KEY__")); + ASSERT_EQ(-1, _row_binlog_tablet->tablet_schema()->field_index("__DORIS_TEST_HIDDEN_VALUE__")); + ASSERT_EQ(-1, _row_binlog_tablet->tablet_schema()->field_index(DELETE_SIGN)); auto partial_update_info = std::make_shared(); ASSERT_TRUE( @@ -280,8 +306,8 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { EXPECT_EQ((std::vector {4, 5}), partial_update_info->missing_cids); RowsetWriterContext row_binlog_context; - row_binlog_context.tablet = _tablet; - row_binlog_context.tablet_schema = _tablet->row_binlog_tablet_schema(); + row_binlog_context.tablet = _row_binlog_tablet; + row_binlog_context.tablet_schema = _row_binlog_tablet->tablet_schema(); row_binlog_context.rowset_state = PREPARED; row_binlog_context.segments_overlap = NONOVERLAPPING; row_binlog_context.max_rows_per_segment = 1024; @@ -303,7 +329,8 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { ASSERT_TRUE(allocate_binlog_lsn(lsn_buffer, 1, *lsn_ids).ok()); binlog_options.insert_seg_lsn(0, lsn_ids); - auto row_binlog_writer_res = _tablet->create_rowset_writer(row_binlog_context, false); + auto row_binlog_writer_res = + _row_binlog_tablet->create_rowset_writer(row_binlog_context, false); ASSERT_TRUE(row_binlog_writer_res.has_value()); auto row_binlog_writer = std::move(row_binlog_writer_res.value()); @@ -314,7 +341,7 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { ASSERT_TRUE(row_binlog_writer->build(row_binlog_rowset).ok()); ASSERT_EQ(1, row_binlog_rowset->num_segments()); - const auto& row_binlog_schema = _tablet->row_binlog_tablet_schema(); + const auto& row_binlog_schema = _row_binlog_tablet->tablet_schema(); ASSERT_EQ(7, row_binlog_schema->num_columns()); std::vector return_columns {0, 1, 2, 3, 4, 5, 6}; RowsetReaderContext reader_context; diff --git a/be/test/storage/compaction/cumulative_compaction_binlog_policy_test.cpp b/be/test/storage/compaction/cumulative_compaction_binlog_policy_test.cpp new file mode 100644 index 00000000000000..4a924f5c4c0224 --- /dev/null +++ b/be/test/storage/compaction/cumulative_compaction_binlog_policy_test.cpp @@ -0,0 +1,364 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/compaction/cumulative_compaction_binlog_policy.h" + +#include +#include +#include +#include + +#include "common/config.h" +#include "gtest/gtest_pred_impl.h" +#include "json2pb/json_to_pb.h" +#include "storage/olap_common.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/storage_engine.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" +#include "util/uid_util.h" + +namespace doris { + +class TestBinlogCumulativeCompactionPolicy : public testing::Test { +public: + TestBinlogCumulativeCompactionPolicy() : _engine(StorageEngine({})) {} + + void SetUp() { + config::binlog_compaction_goal_size_mbytes = 128; + config::binlog_compaction_file_count_threshold = 100; + config::binlog_level_compaction_max_deltas = 2000; + config::binlog_compaction_time_threshold_seconds = 3600; + config::total_permits_for_compaction_score = 1000000; + + _tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F)); + _tablet_meta->set_is_row_binlog_tablet(true); + _tablet_meta->set_compaction_policy(std::string(CUMULATIVE_BINLOG_POLICY)); + + _json_rowset_meta = R"({ + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "BETA_ROWSET", + "rowset_state": "VISIBLE", + "start_version": 2, + "end_version": 2, + "num_rows": 3929, + "total_disk_size": 41, + "data_disk_size": 41, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670, + "num_segments": 1 + })"; + } + void TearDown() {} + + void init_rs_meta(RowsetMetaSharedPtr& pb1, int64_t start, int64_t end, + int8_t compaction_level) { + RowsetMetaPB rowset_meta_pb; + json2pb::JsonToProtoMessage(_json_rowset_meta, &rowset_meta_pb); + rowset_meta_pb.set_start_version(start); + rowset_meta_pb.set_end_version(end); + rowset_meta_pb.set_creation_time(10000); + + pb1->init_from_pb(rowset_meta_pb); + pb1->set_total_disk_size(41); + pb1->set_data_disk_size(41); + pb1->set_segments_overlap(NONOVERLAPPING); + pb1->mark_row_binlog(); + pb1->set_compaction_level(compaction_level); + pb1->set_tablet_schema(_tablet_meta->tablet_schema()); + } + + // LMax rowsets layout: + // + // version: 0 1 2 | 3 4 + // size: - goal goal | - goal + // overlap: - yes(5) - | - - + // enough: Y Y Y | N Y + // ^ + // cumulative point = 3 + // + // [0-0],[1-1],[2-2] are compact-enough -> cumulative point stops at 3. + // [1-1] before the point is overlapping (raw score 5) but is counted as 1, so the score test + // can tell apart "before the point counts as 1" from "after uses the raw compaction score". + // After the point [3-3],[4-4] are non-overlapping (score 1 each), so their physical rewrite + // score (2) is smaller than the quick merge score (3) and the quick merge path is chosen. + void init_rs_meta_lmax(std::vector* rs_metas) { + int64_t goal_size = config::binlog_compaction_goal_size_mbytes * 1024 * 1024; + int8_t max_level = BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1; + + RowsetMetaSharedPtr ptr1(new RowsetMeta()); + init_rs_meta(ptr1, 0, 0, max_level); + rs_metas->push_back(ptr1); + + RowsetMetaSharedPtr ptr2(new RowsetMeta()); + init_rs_meta(ptr2, 1, 1, max_level); + ptr2->set_total_disk_size(goal_size); + ptr2->set_data_disk_size(goal_size); + ptr2->set_num_segments(5); + ptr2->set_segments_overlap(OVERLAPPING); + rs_metas->push_back(ptr2); + + RowsetMetaSharedPtr ptr3(new RowsetMeta()); + init_rs_meta(ptr3, 2, 2, max_level); + ptr3->set_total_disk_size(goal_size); + ptr3->set_data_disk_size(goal_size); + rs_metas->push_back(ptr3); + + RowsetMetaSharedPtr ptr4(new RowsetMeta()); + init_rs_meta(ptr4, 3, 3, max_level); + rs_metas->push_back(ptr4); + + RowsetMetaSharedPtr ptr5(new RowsetMeta()); + init_rs_meta(ptr5, 4, 4, max_level); + ptr5->set_total_disk_size(goal_size); + ptr5->set_data_disk_size(goal_size); + ptr5->set_num_segments(5); + rs_metas->push_back(ptr5); + } + + // L0 rowsets: two overlapping rowsets, score = 4 + 1. + void init_rs_meta_l0(std::vector* rs_metas) { + RowsetMetaSharedPtr ptr1(new RowsetMeta()); + init_rs_meta(ptr1, 0, 0, 0); + ptr1->set_num_segments(4); + ptr1->set_segments_overlap(OVERLAPPING); + rs_metas->push_back(ptr1); + + RowsetMetaSharedPtr ptr2(new RowsetMeta()); + init_rs_meta(ptr2, 1, 1, 0); + rs_metas->push_back(ptr2); + } + +protected: + std::string _json_rowset_meta; + StorageEngine _engine; + TabletMetaSharedPtr _tablet_meta; +}; + +// cumulative point advances over the contiguous compact-enough LMax prefix and stops at +// the first rowset that is not compact-enough. +TEST_F(TestBinlogCumulativeCompactionPolicy, calculate_cumulative_point) { + std::vector rs_metas; + init_rs_meta_lmax(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->calculate_cumulative_point(); + + EXPECT_EQ(3, _tablet->cumulative_layer_point()); +} + +// LMax score: each rowset before cumulative point counts as 1, the others use compaction score. +TEST_F(TestBinlogCumulativeCompactionPolicy, calc_lmax_score) { + std::vector rs_metas; + init_rs_meta_lmax(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->calculate_cumulative_point(); + EXPECT_EQ(3, _tablet->cumulative_layer_point()); + + int8_t max_level = BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1; + // before cumulative point: [0-0] -> 1, [1-1] -> 1 (raw score 5 but counted as 1), [2-2] -> 1; + // after cumulative point: [3-3] -> 1, [4-4] -> 1 (raw score). total = 5. + EXPECT_EQ(5, dynamic_cast( + _tablet->cumulative_compaction_policy()) + ->calc_binlog_compaction_level_score(_tablet.get(), max_level)); +} + +// L0 score always uses the raw compaction score and is independent of cumulative point. +TEST_F(TestBinlogCumulativeCompactionPolicy, calc_l0_score) { + std::vector rs_metas; + init_rs_meta_l0(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->calculate_cumulative_point(); + + int8_t prefer_level = -1; + uint32_t score = + dynamic_cast(_tablet->cumulative_compaction_policy()) + ->calc_binlog_compaction_score(_tablet.get(), &prefer_level); + // L0 rowsets: [0-0] -> 4 (overlapping), [1-1] -> 1. total = 5. + EXPECT_EQ(5, score); + EXPECT_EQ(0, prefer_level); +} + +// Pick level from candidate rowsets. Versions are ordered old -> new as higher -> lower level: +// L2, L2, L1, L1, L0, L0. Even if the whole tablet has a higher L0 score, this round's +// candidate set only contains L1 rowsets, so the policy should choose L1. +TEST_F(TestBinlogCumulativeCompactionPolicy, pick_level_from_candidate_rowsets) { + config::binlog_compaction_file_count_threshold = 2; + + for (int64_t version = 0; version < 2; ++version) { + RowsetMetaSharedPtr rowset(new RowsetMeta()); + init_rs_meta(rowset, version, version, 2); + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + for (int64_t version = 2; version < 4; ++version) { + RowsetMetaSharedPtr rowset(new RowsetMeta()); + init_rs_meta(rowset, version, version, 1); + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + for (int64_t version = 4; version < 6; ++version) { + RowsetMetaSharedPtr rowset(new RowsetMeta()); + init_rs_meta(rowset, version, version, 0); + rowset->set_num_segments(10); + rowset->set_segments_overlap(OVERLAPPING); + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + + std::vector candidate_rowsets; + for (const auto& rs : _tablet->pick_candidate_rowsets_to_binlog_compaction()) { + if (rs->rowset_meta()->compaction_level() == 1) { + candidate_rowsets.push_back(rs); + } + } + + std::vector input_rowsets; + size_t compaction_score = 0; + int picked = + dynamic_cast(_tablet->cumulative_compaction_policy()) + ->pick_input_rowsets(_tablet.get(), candidate_rowsets, + config::binlog_level_compaction_max_deltas, + config::cumulative_compaction_min_deltas, &input_rowsets, + nullptr, &compaction_score); + + EXPECT_EQ(2, picked); + EXPECT_EQ(2, compaction_score); + EXPECT_EQ(2, input_rowsets.size()); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(3, input_rowsets.back()->end_version()); +} + +// LMax quick merge: the physical rewrite over the remaining rowsets is triggered, but the quick +// merge over the compact-enough prefix has a higher score, so the quick merge path is chosen. +TEST_F(TestBinlogCumulativeCompactionPolicy, pick_input_rowsets_lmax_quick_merge) { + std::vector rs_metas; + init_rs_meta_lmax(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->calculate_cumulative_point(); + EXPECT_EQ(3, _tablet->cumulative_layer_point()); + + auto candidate_rowsets = _tablet->pick_candidate_rowsets_to_binlog_compaction(); + + int8_t max_level = BinlogCumulativeCompactionPolicy::kBinlogCompactionMaxLevel - 1; + std::vector input_rowsets; + int picked = + dynamic_cast(_tablet->cumulative_compaction_policy()) + ->pick_input_rowsets(_tablet.get(), candidate_rowsets, max_level, + config::binlog_level_compaction_max_deltas, + &input_rowsets); + + // quick merge score (3) > physical rewrite score (2): [0-0],[1-1],[2-2] are quick merged. + EXPECT_EQ(3, picked); + EXPECT_EQ(3, input_rowsets.size()); + EXPECT_EQ(0, input_rowsets.front()->start_version()); + EXPECT_EQ(2, input_rowsets.back()->end_version()); +} + +// L0 physical rewrite: rowsets are merged when the file count trigger is met. +TEST_F(TestBinlogCumulativeCompactionPolicy, pick_input_rowsets_l0_physical_rewrite) { + config::binlog_compaction_file_count_threshold = 3; + std::vector rs_metas; + init_rs_meta_l0(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->calculate_cumulative_point(); + + auto candidate_rowsets = _tablet->pick_candidate_rowsets_to_binlog_compaction(); + + std::vector input_rowsets; + int picked = + dynamic_cast(_tablet->cumulative_compaction_policy()) + ->pick_input_rowsets(_tablet.get(), candidate_rowsets, 0, + config::binlog_level_compaction_max_deltas, + &input_rowsets); + + // L0 score 4 + 1 = 5 >= file count threshold 3, so both rowsets are physically rewritten. + EXPECT_EQ(2, picked); + EXPECT_EQ(2, input_rowsets.size()); +} + +// L0 not triggered: when size / score / time thresholds are all unmet, nothing is picked. +TEST_F(TestBinlogCumulativeCompactionPolicy, pick_input_rowsets_l0_not_triggered) { + config::binlog_compaction_file_count_threshold = 100; + std::vector rs_metas; + init_rs_meta_l0(&rs_metas); + + for (auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr _tablet(new Tablet(_engine, _tablet_meta, nullptr)); + ASSERT_TRUE(_tablet->init().ok()); + _tablet->set_last_cumu_compaction_success_time(UnixMillis()); + _tablet->calculate_cumulative_point(); + + auto candidate_rowsets = _tablet->pick_candidate_rowsets_to_binlog_compaction(); + + std::vector input_rowsets; + int picked = + dynamic_cast(_tablet->cumulative_compaction_policy()) + ->pick_input_rowsets(_tablet.get(), candidate_rowsets, 0, + config::binlog_level_compaction_max_deltas, + &input_rowsets); + + // L0 score 5 < file count threshold 100, size below goal, time threshold not reached. + EXPECT_EQ(0, picked); + EXPECT_EQ(0, input_rowsets.size()); +} + +} // namespace doris diff --git a/be/test/storage/rowset/rowset_meta_manager_test.cpp b/be/test/storage/rowset/rowset_meta_manager_test.cpp index 5ab24cc2aaadb8..53cc3203aa3025 100644 --- a/be/test/storage/rowset/rowset_meta_manager_test.cpp +++ b/be/test/storage/rowset/rowset_meta_manager_test.cpp @@ -122,13 +122,9 @@ TEST_F(RowsetMetaManagerTest, SaveAndLoad) { auto attach_rowset_meta = create_rowset_meta(20001, RowsetStatePB::COMMITTED, Version {7, 7}, true); - std::map attach_rowset_map; - attach_rowset_map.emplace(attach_rowset_meta->rowset_id(), - to_rowset_meta_pb(attach_rowset_meta)); - auto st = RowsetMetaManager::save(meta(), tablet_uid(), base_rowset_meta->rowset_id(), to_rowset_meta_pb(base_rowset_meta), BinlogFormatPB::ROW, - &attach_rowset_map); + to_rowset_meta_pb(attach_rowset_meta)); ASSERT_TRUE(st.ok()) << st; RowsetMetaSharedPtr loaded_base_meta = std::make_shared(); @@ -139,25 +135,14 @@ TEST_F(RowsetMetaManagerTest, SaveAndLoad) { EXPECT_EQ(loaded_base_meta->tablet_uid().to_string(), tablet_uid().to_string()); EXPECT_EQ(loaded_base_meta->version(), base_rowset_meta->version()); - std::vector> - traversed_attach_metas; - st = RowsetMetaManager::traverse_row_binlog_metas( - meta(), [&traversed_attach_metas]( - const TabletUid& tablet_uid, const RowsetId& base_rowset_id, - const RowsetId& row_binlog_rowset_id, const std::string& value) { - auto rowset_meta = std::make_shared(); - EXPECT_TRUE(rowset_meta->init(value)); - traversed_attach_metas.emplace_back(tablet_uid, base_rowset_id, - row_binlog_rowset_id, std::move(rowset_meta)); - return true; - }); + RowsetMetaSharedPtr loaded_attach_meta = std::make_shared(); + st = RowsetMetaManager::get_rowset_meta(meta(), tablet_uid(), attach_rowset_meta->rowset_id(), + loaded_attach_meta); ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(traversed_attach_metas.size(), 1); - EXPECT_EQ(std::get<0>(traversed_attach_metas[0]).to_string(), tablet_uid().to_string()); - EXPECT_EQ(std::get<1>(traversed_attach_metas[0]), base_rowset_meta->rowset_id()); - EXPECT_EQ(std::get<2>(traversed_attach_metas[0]), attach_rowset_meta->rowset_id()); - EXPECT_EQ(std::get<3>(traversed_attach_metas[0])->rowset_id(), attach_rowset_meta->rowset_id()); - EXPECT_TRUE(std::get<3>(traversed_attach_metas[0])->is_row_binlog()); + EXPECT_EQ(loaded_attach_meta->rowset_id(), attach_rowset_meta->rowset_id()); + EXPECT_EQ(loaded_attach_meta->tablet_uid().to_string(), tablet_uid().to_string()); + EXPECT_EQ(loaded_attach_meta->version(), attach_rowset_meta->version()); + EXPECT_TRUE(loaded_attach_meta->is_row_binlog()); } TEST_F(RowsetMetaManagerTest, Remove) { @@ -165,63 +150,31 @@ TEST_F(RowsetMetaManagerTest, Remove) { auto attach_rowset_meta = create_rowset_meta(20011, RowsetStatePB::VISIBLE, Version {9, 9}, true); - std::map attach_rowset_map; - attach_rowset_map.emplace(attach_rowset_meta->rowset_id(), - to_rowset_meta_pb(attach_rowset_meta)); - auto st = RowsetMetaManager::save(meta(), tablet_uid(), base_rowset_meta->rowset_id(), to_rowset_meta_pb(base_rowset_meta), BinlogFormatPB::ROW, - &attach_rowset_map); - ASSERT_TRUE(st.ok()) << st; - - std::map base_rowset_id_to_row_binlog; - st = RowsetMetaManager::get_row_binlog_base_rowset_ids( - meta(), tablet_uid(), base_rowset_id_to_row_binlog, - std::set {attach_rowset_meta->rowset_id()}); + to_rowset_meta_pb(attach_rowset_meta)); ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(base_rowset_id_to_row_binlog.size(), 1); - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->first, base_rowset_meta->rowset_id()); - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->second, attach_rowset_meta->rowset_id()); - st = RowsetMetaManager::remove_row_binlog(meta(), tablet_uid(), base_rowset_meta->rowset_id(), - attach_rowset_meta->rowset_id()); + st = RowsetMetaManager::exists(meta(), tablet_uid(), attach_rowset_meta->rowset_id()); ASSERT_TRUE(st.ok()) << st; - int traversed_attach_meta_count = 0; - st = RowsetMetaManager::traverse_row_binlog_metas( - meta(), [&traversed_attach_meta_count](const TabletUid&, const RowsetId&, - const RowsetId&, const std::string&) { - ++traversed_attach_meta_count; - return true; - }); + st = RowsetMetaManager::remove(meta(), tablet_uid(), attach_rowset_meta->rowset_id()); ASSERT_TRUE(st.ok()) << st; - EXPECT_EQ(traversed_attach_meta_count, 0); + EXPECT_TRUE(RowsetMetaManager::exists(meta(), tablet_uid(), attach_rowset_meta->rowset_id()) + .is()); auto base_rowset_meta_2 = create_rowset_meta(20012, RowsetStatePB::VISIBLE, Version {10, 10}); auto attach_rowset_meta_2 = create_rowset_meta(20013, RowsetStatePB::VISIBLE, Version {10, 10}, true); - std::map attach_rowset_map_2; - attach_rowset_map_2.emplace(attach_rowset_meta_2->rowset_id(), - to_rowset_meta_pb(attach_rowset_meta_2)); - st = RowsetMetaManager::save(meta(), tablet_uid(), base_rowset_meta_2->rowset_id(), to_rowset_meta_pb(base_rowset_meta_2), BinlogFormatPB::ROW, - &attach_rowset_map_2); - ASSERT_TRUE(st.ok()) << st; - - st = RowsetMetaManager::remove_row_binlog_metas( - meta(), tablet_uid(), std::set {attach_rowset_meta_2->rowset_id()}); + to_rowset_meta_pb(attach_rowset_meta_2)); ASSERT_TRUE(st.ok()) << st; - traversed_attach_meta_count = 0; - st = RowsetMetaManager::traverse_row_binlog_metas( - meta(), [&traversed_attach_meta_count](const TabletUid&, const RowsetId&, - const RowsetId&, const std::string&) { - ++traversed_attach_meta_count; - return true; - }); + st = RowsetMetaManager::remove(meta(), tablet_uid(), attach_rowset_meta_2->rowset_id()); ASSERT_TRUE(st.ok()) << st; - EXPECT_EQ(traversed_attach_meta_count, 0); + EXPECT_TRUE(RowsetMetaManager::exists(meta(), tablet_uid(), attach_rowset_meta_2->rowset_id()) + .is()); } } // namespace doris diff --git a/be/test/storage/snapshot/snapshot_manager_test.cpp b/be/test/storage/snapshot/snapshot_manager_test.cpp index 6ad204d51ee54a..00694eacbc6719 100644 --- a/be/test/storage/snapshot/snapshot_manager_test.cpp +++ b/be/test/storage/snapshot/snapshot_manager_test.cpp @@ -135,7 +135,6 @@ TEST_F(SnapshotManagerTest, TestConvertRowsetIdsNormal) { index_pb->set_index_name("test_index"); index_pb->set_index_type(IndexType::BITMAP); index_pb->add_col_unique_id(0); - tablet_meta_pb.mutable_row_binlog_schema()->CopyFrom(*schema_pb); // rowset meta 0 RowsetMetaPB* rowset_meta = tablet_meta_pb.add_rs_metas(); @@ -272,85 +271,4 @@ TEST_F(SnapshotManagerTest, TestConvertRowsetIdsNormal) { EXPECT_EQ(converted_rowset_index.col_unique_id(0), 0); } -TEST_F(SnapshotManagerTest, TestConvertRowsetIdsRowBinlog) { - std::string clone_dir = _engine_data_path + "/clone_dir_row_binlog"; - std::string row_binlog_dir = fmt::format("{}/{}", clone_dir, FDRowBinlogSuffix); - EXPECT_TRUE(io::global_local_filesystem()->create_directory(clone_dir).ok()); - EXPECT_TRUE(io::global_local_filesystem()->create_directory(row_binlog_dir).ok()); - - TabletMetaPB tablet_meta_pb = testutil::create_tablet_meta_pb(10007, 12347, 1, 1000, 100); - TabletSchemaPB* schema_pb = tablet_meta_pb.mutable_schema(); - schema_pb->set_num_short_key_columns(1); - schema_pb->set_num_rows_per_row_block(1024); - schema_pb->set_compress_kind(COMPRESS_LZ4); - testutil::add_column_pb(schema_pb, 0, "k1", "INT", true, false); - testutil::add_column_pb(schema_pb, 1, "v1", "STRING", false, true); - tablet_meta_pb.mutable_row_binlog_schema()->CopyFrom(*schema_pb); - - RowsetMetaPB* row_binlog_meta = tablet_meta_pb.add_row_binlog_rs_metas(); - row_binlog_meta->set_rowset_id(10002); - row_binlog_meta->set_rowset_id_v2("02000000000000010002"); - row_binlog_meta->set_tablet_id(10007); - row_binlog_meta->set_partition_id(100); - row_binlog_meta->set_tablet_schema_hash(12347); - row_binlog_meta->set_rowset_type(BETA_ROWSET); - row_binlog_meta->set_rowset_state(VISIBLE); - row_binlog_meta->set_start_version(2); - row_binlog_meta->set_end_version(2); - row_binlog_meta->set_num_segments(0); - row_binlog_meta->set_num_rows(100); - row_binlog_meta->set_total_disk_size(1024); - row_binlog_meta->set_data_disk_size(1000); - row_binlog_meta->set_index_disk_size(24); - row_binlog_meta->set_empty(false); - row_binlog_meta->set_is_row_binlog(true); - row_binlog_meta->set_compaction_level(3); - row_binlog_meta->mutable_tablet_schema()->CopyFrom(*schema_pb); - - std::string meta_file = clone_dir + "/" + std::to_string(20007) + ".hdr"; - EXPECT_TRUE(TabletMeta::save(meta_file, tablet_meta_pb).ok()); - - TCreateTabletReq create_tablet_req; - create_tablet_req.__set_tablet_id(20007); - create_tablet_req.__set_replica_id(2); - create_tablet_req.__set_table_id(2000); - create_tablet_req.__set_partition_id(200); - create_tablet_req.__set_version(1); - - TTabletSchema tablet_schema; - tablet_schema.__set_schema_hash(65433); - tablet_schema.__set_short_key_column_count(schema_pb->num_short_key_columns()); - tablet_schema.__set_keys_type(TKeysType::AGG_KEYS); - tablet_schema.__set_storage_type(TStorageType::COLUMN); - tablet_schema.__set_columns( - {testutil::create_tablet_column({"k1", TPrimitiveType::INT, true}), - testutil::create_tablet_column( - {"v1", TPrimitiveType::STRING, false, true, TAggregationType::REPLACE})}); - create_tablet_req.__set_tablet_schema(tablet_schema); - - std::vector stores; - stores.push_back(_data_dir); - RuntimeProfile profile("CreateTablet"); - - Status status = _engine->tablet_manager()->create_tablet(create_tablet_req, stores, &profile); - EXPECT_TRUE(status.ok()) << "Failed to create tablet: " << status; - - auto result = - _engine->snapshot_mgr()->convert_rowset_ids(clone_dir, 20007, 2, 2000, 200, 65433); - EXPECT_TRUE(result.has_value()); - - TabletMetaPB converted_meta_pb; - EXPECT_TRUE(TabletMeta::load_from_file(meta_file, &converted_meta_pb).ok()); - EXPECT_EQ(converted_meta_pb.row_binlog_rs_metas_size(), 1); - - const RowsetMetaPB& converted_row_binlog_meta = converted_meta_pb.row_binlog_rs_metas(0); - EXPECT_EQ(converted_row_binlog_meta.tablet_id(), 20007); - EXPECT_EQ(converted_row_binlog_meta.partition_id(), 200); - EXPECT_NE(converted_row_binlog_meta.rowset_id(), 10002); - EXPECT_NE(converted_row_binlog_meta.rowset_id_v2(), "02000000000000010002"); - EXPECT_EQ(converted_row_binlog_meta.source_rowset_id(), "02000000000000010002"); - EXPECT_EQ(converted_row_binlog_meta.source_tablet_id(), 10007); - EXPECT_TRUE(converted_row_binlog_meta.is_row_binlog()); - EXPECT_EQ(converted_row_binlog_meta.compaction_level(), 3); -} } // namespace doris diff --git a/be/test/storage/txn/txn_manager_test.cpp b/be/test/storage/txn/txn_manager_test.cpp index 30e2e96233e725..0dac20e11d89d0 100644 --- a/be/test/storage/txn/txn_manager_test.cpp +++ b/be/test/storage/txn/txn_manager_test.cpp @@ -406,22 +406,23 @@ TEST_F(TxnManagerTest, PublishVersionWithCommitTSO) { TEST_F(TxnManagerTest, TxnWithRowBinlog) { auto binlog_rowset = create_binlog_rowset(30000, _rowset->version()); - std::vector attach_rowsets {binlog_rowset}; + RowBinlogTxnInfo attach_row_binlog; + attach_row_binlog.rowset = binlog_rowset; + attach_row_binlog.tablet = k_engine->tablet_manager()->get_tablet(tablet_id, _tablet_uid); auto guard = k_engine->pending_local_rowsets().add( {_rowset->rowset_id(), binlog_rowset->rowset_id()}); auto st = k_engine->txn_manager()->commit_txn( _meta.get(), partition_id, transaction_id, tablet_id, _tablet_uid, load_id, _rowset, - std::move(guard), false, nullptr, &attach_rowsets); + std::move(guard), false, nullptr, attach_row_binlog); ASSERT_TRUE(st.ok()) << st; - std::map base_rowset_id_to_row_binlog; - st = RowsetMetaManager::get_row_binlog_base_rowset_ids( - _meta.get(), _tablet_uid, base_rowset_id_to_row_binlog, - std::set {binlog_rowset->rowset_id()}); + RowsetMetaSharedPtr committed_binlog_meta(new RowsetMeta()); + st = RowsetMetaManager::get_rowset_meta(_meta.get(), _tablet_uid, binlog_rowset->rowset_id(), + committed_binlog_meta); ASSERT_TRUE(st.ok()) << st; - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->first, _rowset->rowset_id()); - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->second, binlog_rowset->rowset_id()); + EXPECT_EQ(committed_binlog_meta->rowset_state(), RowsetStatePB::COMMITTED); + EXPECT_TRUE(committed_binlog_meta->is_row_binlog()); Version new_version(10, 10); TabletPublishStatistics stats; @@ -437,24 +438,13 @@ TEST_F(TxnManagerTest, TxnWithRowBinlog) { EXPECT_EQ(rowset_meta->start_version(), 10); EXPECT_EQ(rowset_meta->end_version(), 10); - bool found_binlog_rowset = false; - st = RowsetMetaManager::traverse_row_binlog_metas( - _meta.get(), [&](const TabletUid&, const RowsetId& base_rowset_id, - const RowsetId& row_binlog_rowset_id, const std::string& value) { - if (row_binlog_rowset_id != binlog_rowset->rowset_id()) { - return true; - } - found_binlog_rowset = true; - EXPECT_EQ(base_rowset_id, _rowset->rowset_id()); - RowsetMetaSharedPtr binlog_rowset_meta(new RowsetMeta()); - EXPECT_TRUE(binlog_rowset_meta->init(value)); - EXPECT_EQ(binlog_rowset_meta->version(), new_version); - EXPECT_EQ(binlog_rowset_meta->rowset_state(), RowsetStatePB::VISIBLE); - EXPECT_TRUE(binlog_rowset_meta->is_row_binlog()); - return true; - }); - ASSERT_TRUE(st.ok()) << st; - EXPECT_TRUE(found_binlog_rowset); + RowsetMetaSharedPtr binlog_rowset_meta(new RowsetMeta()); + st = RowsetMetaManager::get_rowset_meta(_meta.get(), _tablet_uid, binlog_rowset->rowset_id(), + binlog_rowset_meta); + ASSERT_TRUE(st.ok()) << st; + EXPECT_EQ(binlog_rowset_meta->version(), new_version); + EXPECT_EQ(binlog_rowset_meta->rowset_state(), RowsetStatePB::VISIBLE); + EXPECT_TRUE(binlog_rowset_meta->is_row_binlog()); } // 1. publish version failed if not found related txn and rowset @@ -501,22 +491,23 @@ TEST_F(TxnManagerTest, DeleteCommittedTxn) { TEST_F(TxnManagerTest, DeleteCommittedTxnWithBinlogRowset) { auto binlog_rowset = create_binlog_rowset(30002, _rowset->version()); - std::vector attach_rowsets {binlog_rowset}; + RowBinlogTxnInfo attach_row_binlog; + attach_row_binlog.rowset = binlog_rowset; + attach_row_binlog.tablet = k_engine->tablet_manager()->get_tablet(tablet_id, _tablet_uid); auto guard = k_engine->pending_local_rowsets().add( {_rowset->rowset_id(), binlog_rowset->rowset_id()}); auto st = k_engine->txn_manager()->commit_txn( _meta.get(), partition_id, transaction_id, tablet_id, _tablet_uid, load_id, _rowset, - std::move(guard), false, nullptr, &attach_rowsets); + std::move(guard), false, nullptr, attach_row_binlog); ASSERT_TRUE(st.ok()) << st; - std::map base_rowset_id_to_row_binlog; - st = RowsetMetaManager::get_row_binlog_base_rowset_ids( - _meta.get(), _tablet_uid, base_rowset_id_to_row_binlog, - std::set {binlog_rowset->rowset_id()}); + RowsetMetaSharedPtr committed_binlog_meta(new RowsetMeta()); + st = RowsetMetaManager::get_rowset_meta(_meta.get(), _tablet_uid, binlog_rowset->rowset_id(), + committed_binlog_meta); ASSERT_TRUE(st.ok()) << st; - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->first, _rowset->rowset_id()); - EXPECT_EQ(base_rowset_id_to_row_binlog.begin()->second, binlog_rowset->rowset_id()); + EXPECT_EQ(committed_binlog_meta->rowset_state(), RowsetStatePB::COMMITTED); + EXPECT_TRUE(committed_binlog_meta->is_row_binlog()); st = k_engine->txn_manager()->delete_txn(_meta.get(), partition_id, transaction_id, tablet_id, _tablet_uid); @@ -525,8 +516,8 @@ TEST_F(TxnManagerTest, DeleteCommittedTxnWithBinlogRowset) { EXPECT_TRUE(RowsetMetaManager::exists(_meta.get(), _tablet_uid, _rowset->rowset_id()) .is()); - EXPECT_FALSE(RowsetMetaManager::row_binlog_meta_exists(_meta.get(), _tablet_uid, - binlog_rowset->rowset_id())); + EXPECT_TRUE(RowsetMetaManager::exists(_meta.get(), _tablet_uid, binlog_rowset->rowset_id()) + .is()); } TEST_F(TxnManagerTest, TabletVersionCache) { diff --git a/be/test/testutil/creators.h b/be/test/testutil/creators.h index ff20413c70e7b8..6756a62dfb2c1c 100644 --- a/be/test/testutil/creators.h +++ b/be/test/testutil/creators.h @@ -194,24 +194,31 @@ inline TCreateTabletReq create_tablet_request( return request; } -inline void enable_row_binlog(TCreateTabletReq* request, int32_t row_binlog_schema_hash = 0) { +inline void enable_row_binlog(TCreateTabletReq* request) { DCHECK(request != nullptr); TBinlogConfig binlog_config; binlog_config.__set_enable(true); binlog_config.__set_binlog_format(TBinlogFormat::ROW); request->__set_binlog_config(binlog_config); +} - TTabletSchema row_binlog_schema = request->tablet_schema; - row_binlog_schema.schema_hash = row_binlog_schema_hash > 0 - ? row_binlog_schema_hash - : request->tablet_schema.schema_hash + 1; +inline TTabletSchema create_row_binlog_tablet_schema(const TTabletSchema& base_schema, + int32_t row_binlog_schema_hash) { + TTabletSchema row_binlog_schema = base_schema; + row_binlog_schema.schema_hash = row_binlog_schema_hash; row_binlog_schema.keys_type = TKeysType::DUP_KEYS; - for (auto& col : row_binlog_schema.columns) { + row_binlog_schema.columns.clear(); + for (const auto& base_col : base_schema.columns) { + if (base_col.__isset.visible && !base_col.visible && !base_col.is_key) { + continue; + } + auto col = base_col; if (!col.is_key) { col.__set_aggregation_type(TAggregationType::NONE); } + row_binlog_schema.columns.push_back(col); } row_binlog_schema.columns.push_back(create_tablet_column( @@ -223,7 +230,7 @@ inline void enable_row_binlog(TCreateTabletReq* request, int32_t row_binlog_sche row_binlog_schema.columns.push_back(create_tablet_column( {BINLOG_OP_COL, TPrimitiveType::BIGINT, false, true, TAggregationType::NONE})); row_binlog_schema.__set_binlog_op_idx(row_binlog_schema.columns.size() - 1); - request->__set_row_binlog_schema(row_binlog_schema); + return row_binlog_schema; } inline TDescriptorTable create_descriptor_table( @@ -274,7 +281,7 @@ inline std::shared_ptr create_table_schema_param( row_binlog_index_schema.columns.push_back(col.column_name); } } - tschema.__set_row_binlog_index_schema(row_binlog_index_schema); + tschema.__set_row_binlog_index_schemas({row_binlog_index_schema}); } Status st = param->init(tschema); EXPECT_TRUE(st.ok()) << st; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 2fe48980f6f193..54c5e5a165f1e7 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -2655,60 +2655,21 @@ bool check_recycle_rowset_key(Transaction* txn, const std::string& recycle_rs_ke return true; } -/** - * 1. Check and confirm tmp rowset kv does not exist - * a. if exist - * 1. if tmp rowset is same with self, it may be a redundant - * retry request, return ok - * 2. else, abort commit_rowset - * b. else, goto 2 - * 2. Remove recycle rowset kv and put tmp rowset kv - */ -void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controller, - const CreateRowsetRequest* request, - CreateRowsetResponse* response, - ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_rowset, get, put, del); - if (!request->has_rowset_meta()) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "no rowset meta"; - return; - } - instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); - if (instance_id.empty()) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "empty instance_id"; - LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id(); - return; - } - doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta()); - if (!rowset_meta.has_tablet_schema() && !rowset_meta.has_schema_version()) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "rowset_meta must have either schema or schema_version"; - return; - } - RPC_RATE_LIMIT(commit_rowset) - +void MetaServiceImpl::commit_rowset_meta(Transaction* txn, const std::string& instance_id, + const std::string& tablet_job_id, + doris::RowsetMetaCloudPB& rowset_meta, + doris::RowsetMetaCloudPB* existed_rowset_meta, + MetaServiceCode& code, std::string& msg) { int64_t tablet_id = rowset_meta.tablet_id(); const auto& rowset_id = rowset_meta.rowset_id_v2(); - auto tmp_rs_key = meta_rowset_tmp_key({instance_id, rowset_meta.txn_id(), tablet_id}); - - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = "failed to create txn"; - return; - } - bool is_versioned_read = is_version_read_enabled(instance_id); auto recycle_rs_key = recycle_rowset_key({instance_id, tablet_id, rowset_id}); if (!config::enable_recycle_delete_rowset_key_check) { - std::string tablet_job_id = request->tablet_job_id(); if (config::enable_tablet_job_check && !tablet_job_id.empty()) { - if (!check_job_existed(txn.get(), code, msg, instance_id, tablet_id, rowset_id, - tablet_job_id, is_versioned_read, resource_mgr_.get())) { + if (!check_job_existed(txn, code, msg, instance_id, tablet_id, rowset_id, tablet_job_id, + is_versioned_read, resource_mgr_.get())) { return; } } @@ -2719,7 +2680,7 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle // If the rowset had load id, it means it is a load request, otherwise it is a // compaction/sc request. if (config::enable_load_txn_status_check && rowset_meta.has_load_id() && - !check_transaction_status(TxnStatusPB::TXN_STATUS_PREPARED, txn.get(), instance_id, + !check_transaction_status(TxnStatusPB::TXN_STATUS_PREPARED, txn, instance_id, rowset_meta.txn_id(), code, msg)) { LOG(WARNING) << "commit rowset failed, txn_id=" << rowset_meta.txn_id() << ", tablet_id=" << tablet_id << ", rowset_id=" << rowset_id @@ -2730,12 +2691,12 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle // Check if commit key already exists. std::string existed_commit_val; - err = txn->get(tmp_rs_key, &existed_commit_val); + TxnErrorCode err = txn->get(tmp_rs_key, &existed_commit_val); if (err == TxnErrorCode::TXN_OK) { if (config::enable_recycle_delete_rowset_key_check) { bool recycle_rs_key_exists = false; - if (!check_recycle_rowset_key(txn.get(), recycle_rs_key, rowset_meta, - &recycle_rs_key_exists, code, msg)) { + if (!check_recycle_rowset_key(txn, recycle_rs_key, rowset_meta, &recycle_rs_key_exists, + code, msg)) { return; } if (recycle_rs_key_exists) { @@ -2752,55 +2713,55 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle return; } } - auto existed_rowset_meta = response->mutable_existed_rowset_meta(); - if (!existed_rowset_meta->ParseFromString(existed_commit_val)) { + + doris::RowsetMetaCloudPB existed_meta; + if (!existed_meta.ParseFromString(existed_commit_val)) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; msg = fmt::format("malformed rowset meta value. key={}", hex(tmp_rs_key)); return; } - if (existed_rowset_meta->rowset_id_v2() == rowset_meta.rowset_id_v2()) { + if (existed_meta.rowset_id_v2() == rowset_meta.rowset_id_v2()) { // Same request, return OK - response->set_allocated_existed_rowset_meta(nullptr); return; } - if (!existed_rowset_meta->has_index_id()) { + if (!existed_meta.has_index_id()) { if (rowset_meta.has_index_id()) { - existed_rowset_meta->set_index_id(rowset_meta.index_id()); + existed_meta.set_index_id(rowset_meta.index_id()); } else if (!is_versioned_read) { TabletIndexPB tablet_idx; - get_tablet_idx(code, msg, txn.get(), instance_id, rowset_meta.tablet_id(), - tablet_idx); + get_tablet_idx(code, msg, txn, instance_id, rowset_meta.tablet_id(), tablet_idx); if (code != MetaServiceCode::OK) return; - existed_rowset_meta->set_index_id(tablet_idx.index_id()); + existed_meta.set_index_id(tablet_idx.index_id()); } else { CloneChainReader reader(instance_id, resource_mgr_.get()); TabletIndexPB tablet_idx; - TxnErrorCode err = reader.get_tablet_index(txn.get(), tablet_id, &tablet_idx); - if (err != TxnErrorCode::TXN_OK) { - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND + TxnErrorCode idx_err = reader.get_tablet_index(txn, tablet_id, &tablet_idx); + if (idx_err != TxnErrorCode::TXN_OK) { + code = idx_err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND - : cast_as(err); + : cast_as(idx_err); msg = fmt::format("failed to get tablet index, tablet_id={}, err={}", tablet_id, - err); + idx_err); LOG(WARNING) << msg; return; } - existed_rowset_meta->set_index_id(tablet_idx.index_id()); + existed_meta.set_index_id(tablet_idx.index_id()); } } - if (!existed_rowset_meta->has_tablet_schema()) { - set_schema_in_existed_rowset(code, msg, txn.get(), instance_id, rowset_meta, - *existed_rowset_meta, is_versioned_read, - resource_mgr_.get()); + if (!existed_meta.has_tablet_schema()) { + set_schema_in_existed_rowset(code, msg, txn, instance_id, rowset_meta, existed_meta, + is_versioned_read, resource_mgr_.get()); if (code != MetaServiceCode::OK) return; } else { - existed_rowset_meta->set_schema_version( - existed_rowset_meta->tablet_schema().schema_version()); + existed_meta.set_schema_version(existed_meta.tablet_schema().schema_version()); } - if (existed_rowset_meta->has_variant_type_in_schema()) { - fill_schema_from_dict(code, msg, instance_id, txn.get(), existed_rowset_meta); + if (existed_meta.has_variant_type_in_schema()) { + fill_schema_from_dict(code, msg, instance_id, txn, &existed_meta); if (code != MetaServiceCode::OK) return; } + if (existed_rowset_meta != nullptr) { + existed_rowset_meta->CopyFrom(existed_meta); + } code = MetaServiceCode::ALREADY_EXISTED; msg = "rowset already exists"; return; @@ -2810,10 +2771,11 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle msg = fmt::format("failed to check whether rowset exists, err={}", err); return; } + if (config::enable_recycle_delete_rowset_key_check) { bool recycle_rs_key_exists = false; - if (!check_recycle_rowset_key(txn.get(), recycle_rs_key, rowset_meta, - &recycle_rs_key_exists, code, msg)) { + if (!check_recycle_rowset_key(txn, recycle_rs_key, rowset_meta, &recycle_rs_key_exists, + code, msg)) { return; } if (!recycle_rs_key_exists) { @@ -2825,23 +2787,25 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle return; } } + // write schema kv if rowset_meta has schema if (config::write_schema_kv && rowset_meta.has_tablet_schema()) { if (!rowset_meta.has_index_id() && !is_versioned_read) { TabletIndexPB tablet_idx; - get_tablet_idx(code, msg, txn.get(), instance_id, rowset_meta.tablet_id(), tablet_idx); + get_tablet_idx(code, msg, txn, instance_id, rowset_meta.tablet_id(), tablet_idx); if (code != MetaServiceCode::OK) return; rowset_meta.set_index_id(tablet_idx.index_id()); } else if (!rowset_meta.has_index_id()) { CloneChainReader reader(instance_id, resource_mgr_.get()); TabletIndexPB tablet_idx; - TxnErrorCode err = - reader.get_tablet_index(txn.get(), rowset_meta.tablet_id(), &tablet_idx); - if (err != TxnErrorCode::TXN_OK) { - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND - : cast_as(err); + TxnErrorCode idx_err = + reader.get_tablet_index(txn, rowset_meta.tablet_id(), &tablet_idx); + if (idx_err != TxnErrorCode::TXN_OK) { + code = idx_err == TxnErrorCode::TXN_KEY_NOT_FOUND + ? MetaServiceCode::TABLET_NOT_FOUND + : cast_as(idx_err); msg = fmt::format("failed to get tablet index, tablet_id={}, err={}", - rowset_meta.tablet_id(), err); + rowset_meta.tablet_id(), idx_err); LOG(WARNING) << msg; return; } @@ -2851,18 +2815,18 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle DCHECK_GE(rowset_meta.tablet_schema().schema_version(), 0); rowset_meta.set_schema_version(rowset_meta.tablet_schema().schema_version()); if (rowset_meta.has_variant_type_in_schema()) { - write_schema_dict(code, msg, instance_id, txn.get(), &rowset_meta); + write_schema_dict(code, msg, instance_id, txn, &rowset_meta); if (code != MetaServiceCode::OK) return; } bool is_versioned_write = is_version_write_enabled(instance_id); std::string schema_key = meta_schema_key( {instance_id, rowset_meta.index_id(), rowset_meta.schema_version()}); - put_schema_kv(code, msg, txn.get(), schema_key, rowset_meta.tablet_schema()); + put_schema_kv(code, msg, txn, schema_key, rowset_meta.tablet_schema()); if (code != MetaServiceCode::OK) return; if (is_versioned_write) { std::string versioned_schema_key = versioned::meta_schema_key( {instance_id, rowset_meta.index_id(), rowset_meta.schema_version()}); - put_versioned_schema_kv(code, msg, txn.get(), versioned_schema_key, + put_versioned_schema_kv(code, msg, txn, versioned_schema_key, rowset_meta.tablet_schema()); if (code != MetaServiceCode::OK) return; } @@ -2885,7 +2849,59 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta); LOG(INFO) << "put tmp_rs_key " << hex(tmp_rs_key) << " delete recycle_rs_key " << hex(recycle_rs_key) << " value_size " << tmp_rs_val.size() << " txn_id " - << request->txn_id() << " segment_key_bounds_bytes " << segment_key_bounds_bytes; + << rowset_meta.txn_id() << " segment_key_bounds_bytes " << segment_key_bounds_bytes; +} + +/** + * 1. Check and confirm tmp rowset kv does not exist + * a. if exist + * 1. if tmp rowset is same with self, it may be a redundant + * retry request, return ok + * 2. else, abort commit_rowset + * b. else, goto 2 + * 2. Remove recycle rowset kv and put tmp rowset kv + */ +void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controller, + const CreateRowsetRequest* request, + CreateRowsetResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(commit_rowset, get, put, del); + if (!request->has_rowset_meta()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no rowset meta"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "empty instance_id"; + LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id(); + return; + } + doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta()); + if (!rowset_meta.has_tablet_schema() && !rowset_meta.has_schema_version()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "rowset_meta must have either schema or schema_version"; + return; + } + RPC_RATE_LIMIT(commit_rowset) + + int64_t tablet_id = rowset_meta.tablet_id(); + const auto& rowset_id = rowset_meta.rowset_id_v2(); + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create txn"; + return; + } + + commit_rowset_meta(txn.get(), instance_id, request->tablet_job_id(), rowset_meta, + response->mutable_existed_rowset_meta(), code, msg); + if (code == MetaServiceCode::ALREADY_EXISTED) return; + response->set_allocated_existed_rowset_meta(nullptr); + if (code != MetaServiceCode::OK) return; + + std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta); err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2907,16 +2923,21 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle } } -void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* controller, - const CreateRowsetRequest* request, - CreateRowsetResponse* response, - ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_tmp_rowset, get, put); +void MetaServiceImpl::commit_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, + CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(commit_rowset, get, put, del); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; return; } + if (!request->has_attach_row_binlog()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no attach row binlog"; + return; + } instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2930,7 +2951,103 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr msg = "rowset_meta must have either schema or schema_version"; return; } - RPC_RATE_LIMIT(update_tmp_rowset) + doris::RowsetMetaCloudPB attach_row_binlog(request->attach_row_binlog()); + if (!attach_row_binlog.has_tablet_schema() && !attach_row_binlog.has_schema_version()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "attach_row_binlog must have either schema or schema_version"; + return; + } + RPC_RATE_LIMIT(commit_rowset) + + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create txn"; + return; + } + + { + MetaServiceCode commit_code = MetaServiceCode::OK; + std::string commit_msg; + commit_rowset_meta(txn.get(), instance_id, request->tablet_job_id(), rowset_meta, + response->mutable_existed_rowset_meta(), commit_code, commit_msg); + if (commit_code != MetaServiceCode::OK && commit_code != MetaServiceCode::ALREADY_EXISTED) { + code = commit_code; + msg = commit_msg; + return; + } + if (commit_code != MetaServiceCode::ALREADY_EXISTED) { + response->set_allocated_existed_rowset_meta(nullptr); + } + code = commit_code; + msg = commit_msg; + } + + { + MetaServiceCode commit_code = MetaServiceCode::OK; + std::string commit_msg; + commit_rowset_meta(txn.get(), instance_id, request->tablet_job_id(), attach_row_binlog, + response->mutable_existed_attach_row_binlog(), commit_code, commit_msg); + if (commit_code != MetaServiceCode::OK && commit_code != MetaServiceCode::ALREADY_EXISTED) { + code = commit_code; + msg = commit_msg; + return; + } + if (commit_code != MetaServiceCode::ALREADY_EXISTED) { + response->set_allocated_existed_attach_row_binlog(nullptr); + } + + // Data rowset and row-binlog rowset are written in the same + // transaction. A normal retry should observe both as existing or both + // as missing. Mismatch means the paired tmp rowset state is already + // inconsistent. + bool rowset_already_exists = code == MetaServiceCode::ALREADY_EXISTED; + bool binlog_already_exists = commit_code == MetaServiceCode::ALREADY_EXISTED; + if (rowset_already_exists != binlog_already_exists) { + msg = fmt::format( + "rowset and attach row binlog existence mismatch, rowset_code={}, " + "binlog_code={}", + MetaServiceCode_Name(code), MetaServiceCode_Name(commit_code)); + code = MetaServiceCode::INVALID_ARGUMENT; + return; + } + if (rowset_already_exists) { + msg = "rowsets already exist"; + return; + } + } + + std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta) + + get_segments_key_bounds_bytes(attach_row_binlog); + std::size_t rowsets_meta_bytes = rowset_meta.ByteSizeLong() + attach_row_binlog.ByteSizeLong(); + err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to save rowset meta, err=" << err; + if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) { + LOG(WARNING) << "failed to commit rowsets, err=value too large" + << ", txn_id=" << request->txn_id() + << ", tablet_id=" << rowset_meta.tablet_id() + << ", rowset_id=" << rowset_meta.rowset_id_v2() + << ", rowsets_meta_bytes=" << rowsets_meta_bytes + << ", segment_key_bounds_bytes=" << segment_key_bounds_bytes + << ", num_segments=" + << rowset_meta.num_segments() + attach_row_binlog.num_segments() + << ", attach_row_binlog_tablet_id=" << attach_row_binlog.tablet_id() + << ", attach_row_binlog_rowset_id=" << attach_row_binlog.rowset_id_v2(); + ss << ". The key column data is too large, or too many partitions are being loaded " + "simultaneously. Please reduce the size of the key column data or lower the " + "number of partitions involved in a single load or update."; + } + msg = ss.str(); + return; + } +} + +void MetaServiceImpl::update_tmp_rowset_meta(Transaction* txn, const std::string& instance_id, + doris::RowsetMetaCloudPB& rowset_meta, + doris::RowsetMetaCloudPB* existed_rowset_meta, + MetaServiceCode& code, std::string& msg) { int64_t tablet_id = rowset_meta.tablet_id(); std::string update_key; @@ -2940,19 +3057,12 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr MetaRowsetTmpKeyInfo key_info {instance_id, txn_id, tablet_id}; meta_rowset_tmp_key(key_info, &update_key); - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = "failed to create txn"; - return; - } - // Check if commit key already exists. std::string existed_commit_val; - err = txn->get(update_key, &existed_commit_val); + TxnErrorCode err = txn->get(update_key, &existed_commit_val); if (err == TxnErrorCode::TXN_OK) { - auto existed_rowset_meta = response->mutable_existed_rowset_meta(); - if (!existed_rowset_meta->ParseFromString(existed_commit_val)) { + if (existed_rowset_meta != nullptr && + !existed_rowset_meta->ParseFromString(existed_commit_val)) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; msg = fmt::format("malformed rowset meta value. key={}", hex(update_key)); return; @@ -2977,7 +3087,7 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr return; } if (rowset_meta.has_variant_type_in_schema()) { - write_schema_dict(code, msg, instance_id, txn.get(), &rowset_meta); + write_schema_dict(code, msg, instance_id, txn, &rowset_meta); if (code != MetaServiceCode::OK) return; } DCHECK_GT(rowset_meta.txn_expiration(), 0); @@ -2992,12 +3102,52 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr LOG(INFO) << "xxx put " << "update_rowset_key " << hex(update_key) << " value_size " << update_val.size() << " segment_key_bounds_bytes " << segment_key_bounds_bytes; +} + +void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* controller, + const CreateRowsetRequest* request, + CreateRowsetResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(update_tmp_rowset, get, put); + if (!request->has_rowset_meta()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no rowset meta"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "empty instance_id"; + LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id(); + return; + } + doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta()); + if (!rowset_meta.has_tablet_schema() && !rowset_meta.has_schema_version()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "rowset_meta must have either schema or schema_version"; + return; + } + RPC_RATE_LIMIT(update_tmp_rowset) + + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create txn"; + return; + } + + update_tmp_rowset_meta(txn.get(), instance_id, rowset_meta, + response->mutable_existed_rowset_meta(), code, msg); + if (code != MetaServiceCode::OK) return; + err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); ss << "failed to update rowset meta, err=" << err; if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) { const auto& rowset_id = rowset_meta.rowset_id_v2(); + int64_t tablet_id = rowset_meta.tablet_id(); + std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta); LOG(WARNING) << "failed to update tmp rowset, err=value too large" << ", txn_id=" << request->txn_id() << ", tablet_id=" << tablet_id << ", rowset_id=" << rowset_id @@ -3013,6 +3163,84 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr } } +void MetaServiceImpl::update_tmp_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, + CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) { + RPC_PREPROCESS(update_tmp_rowset, get, put); + if (!request->has_rowset_meta()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no rowset meta"; + return; + } + if (!request->has_attach_row_binlog()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no attach row binlog"; + return; + } + instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); + if (instance_id.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "empty instance_id"; + LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id(); + return; + } + doris::RowsetMetaCloudPB rowset_meta(request->rowset_meta()); + if (!rowset_meta.has_tablet_schema() && !rowset_meta.has_schema_version()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "rowset_meta must have either schema or schema_version"; + return; + } + doris::RowsetMetaCloudPB attach_row_binlog(request->attach_row_binlog()); + if (!attach_row_binlog.has_tablet_schema() && !attach_row_binlog.has_schema_version()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "attach_row_binlog must have either schema or schema_version"; + return; + } + RPC_RATE_LIMIT(update_tmp_rowset) + + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create txn"; + return; + } + + update_tmp_rowset_meta(txn.get(), instance_id, rowset_meta, + response->mutable_existed_rowset_meta(), code, msg); + if (code != MetaServiceCode::OK) return; + update_tmp_rowset_meta(txn.get(), instance_id, attach_row_binlog, + response->mutable_existed_attach_row_binlog(), code, msg); + if (code != MetaServiceCode::OK) return; + + err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to update rowsets meta, err=" << err; + if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) { + std::size_t segment_key_bounds_bytes = get_segments_key_bounds_bytes(rowset_meta) + + get_segments_key_bounds_bytes(attach_row_binlog); + std::size_t rowsets_meta_bytes = + rowset_meta.ByteSizeLong() + attach_row_binlog.ByteSizeLong(); + LOG(WARNING) << "failed to update tmp rowsets, err=value too large" + << ", txn_id=" << request->txn_id() + << ", tablet_id=" << rowset_meta.tablet_id() + << ", rowset_id=" << rowset_meta.rowset_id_v2() + << ", rowsets_meta_bytes=" << rowsets_meta_bytes + << ", segment_key_bounds_bytes=" << segment_key_bounds_bytes + << ", num_segments=" + << rowset_meta.num_segments() + attach_row_binlog.num_segments() + << ", attach_row_binlog_tablet_id=" << attach_row_binlog.tablet_id() + << ", attach_row_binlog_rowset_id=" << attach_row_binlog.rowset_id_v2(); + ss << ". The key column data is too large, or too many partitions are being loaded " + "simultaneously. Please reduce the size of the key column data or lower the " + "number of partitions involved in a single load or update."; + } + msg = ss.str(); + return; + } +} + void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, const std::string& instance_id, int64_t tablet_id, MetaServiceCode& code, std::string& msg, GetRowsetResponse* response) { diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index a0594a945d6f6f..4b9cdf18e5d28e 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -184,10 +184,18 @@ class MetaServiceImpl : public cloud::MetaService { const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) override; + void commit_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) override; + void update_tmp_rowset(::google::protobuf::RpcController* controller, const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) override; + void update_tmp_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) override; + void update_packed_file_info(::google::protobuf::RpcController* controller, const UpdatePackedFileInfoRequest* request, UpdatePackedFileInfoResponse* response, @@ -429,6 +437,15 @@ class MetaServiceImpl : public cloud::MetaService { ::google::protobuf::Closure* done) override; private: + void commit_rowset_meta(Transaction* txn, const std::string& instance_id, + const std::string& tablet_job_id, doris::RowsetMetaCloudPB& rowset_meta, + doris::RowsetMetaCloudPB* existed_rowset_meta, MetaServiceCode& code, + std::string& msg); + void update_tmp_rowset_meta(Transaction* txn, const std::string& instance_id, + doris::RowsetMetaCloudPB& rowset_meta, + doris::RowsetMetaCloudPB* existed_rowset_meta, + MetaServiceCode& code, std::string& msg); + std::pair alter_instance( const AlterInstanceRequest* request, std::function(Transaction*, InstanceInfoPB*)> @@ -662,12 +679,24 @@ class MetaServiceProxy final : public MetaService { call_impl(&cloud::MetaService::commit_rowset, controller, request, response, done); } + void commit_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) override { + call_impl(&cloud::MetaService::commit_rowsets, controller, request, response, done); + } + void update_tmp_rowset(::google::protobuf::RpcController* controller, const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) override { call_impl(&cloud::MetaService::update_tmp_rowset, controller, request, response, done); } + void update_tmp_rowsets(::google::protobuf::RpcController* controller, + const CreateRowsetsRequest* request, CreateRowsetsResponse* response, + ::google::protobuf::Closure* done) override { + call_impl(&cloud::MetaService::update_tmp_rowsets, controller, request, response, done); + } + void update_packed_file_info(::google::protobuf::RpcController* controller, const UpdatePackedFileInfoRequest* request, UpdatePackedFileInfoResponse* response, diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 5c35958b1f0b83..2dc7cce8b399e2 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -1528,6 +1528,7 @@ void MetaServiceImpl::commit_txn_immediately( TxnErrorCode& err, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); + int64_t commit_tso = request->has_commit_tso() ? request->commit_tso() : -1; bool is_versioned_write = is_version_write_enabled(instance_id); bool is_versioned_read = is_version_read_enabled(instance_id); @@ -1733,6 +1734,8 @@ void MetaServiceImpl::commit_txn_immediately( i.set_start_version(new_version); i.set_end_version(new_version); i.set_visible_ts_ms(rowsets_visible_ts_ms); + i.mutable_commit_tso()->set_start_tso(commit_tso); + i.mutable_commit_tso()->set_end_tso(commit_tso); // Accumulate affected rows auto& stats = tablet_stats[tablet_id]; @@ -1896,6 +1899,7 @@ void MetaServiceImpl::commit_txn_immediately( } txn_info.set_commit_time(commit_time); txn_info.set_finish_time(commit_time); + txn_info.set_commit_tso(commit_tso); if (request->has_commit_attachment()) { txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment()); } @@ -2204,6 +2208,7 @@ void MetaServiceImpl::commit_txn_eventually( std::stringstream ss; TxnErrorCode err = TxnErrorCode::TXN_OK; int64_t txn_id = request->txn_id(); + int64_t commit_tso = request->has_commit_tso() ? request->commit_tso() : -1; bool is_versioned_write = is_version_write_enabled(instance_id); bool is_versioned_read = is_version_read_enabled(instance_id); @@ -2428,6 +2433,7 @@ void MetaServiceImpl::commit_txn_eventually( } txn_info.set_commit_time(commit_time); txn_info.set_finish_time(commit_time); + txn_info.set_commit_tso(commit_tso); if (request->has_commit_attachment()) { txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment()); } @@ -2723,6 +2729,7 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, int64_t db_id, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); + int64_t commit_tso = request->has_commit_tso() ? request->commit_tso() : -1; auto sub_txn_infos = request->sub_txn_infos(); std::map>> sub_txn_to_tmp_rowsets_meta; @@ -2934,6 +2941,8 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, i.set_start_version(new_version); i.set_end_version(new_version); i.set_visible_ts_ms(rowsets_visible_ts_ms); + i.mutable_commit_tso()->set_start_tso(commit_tso); + i.mutable_commit_tso()->set_end_tso(commit_tso); LOG(INFO) << "xxx update rowset version, txn_id=" << txn_id << ", sub_txn_id=" << sub_txn_id << ", table_id=" << table_id << ", partition_id=" << partition_id << ", tablet_id=" << tablet_id @@ -3101,6 +3110,7 @@ void MetaServiceImpl::commit_txn_with_sub_txn(const CommitTxnRequest* request, } txn_info.set_commit_time(commit_time); txn_info.set_finish_time(commit_time); + txn_info.set_commit_tso(commit_tso); if (request->has_commit_attachment()) { txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment()); } diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 308dfd2fc4d908..bec4b13f068638 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -213,7 +213,7 @@ void convert_tmp_rowsets( std::vector>& tmp_rowsets_meta, std::map& tablet_ids, bool is_versioned_write, bool is_versioned_read, Versionstamp versionstamp, ResourceManager* resource_mgr, - bool defer_deleting_pending_delete_bitmaps) { + bool defer_deleting_pending_delete_bitmaps, int64_t commit_tso) { std::stringstream ss; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -390,6 +390,8 @@ void convert_tmp_rowsets( tmp_rowset_pb.set_start_version(version); tmp_rowset_pb.set_end_version(version); tmp_rowset_pb.set_visible_ts_ms(rowsets_visible_ts_ms); + tmp_rowset_pb.mutable_commit_tso()->set_start_tso(commit_tso); + tmp_rowset_pb.mutable_commit_tso()->set_end_tso(commit_tso); rowset_val.clear(); if (!tmp_rowset_pb.SerializeToString(&rowset_val)) { @@ -733,10 +735,10 @@ void TxnLazyCommitTask::commit() { fmt::format("txn_{}_parallel_commit", txn_id_)); for (int64_t partition_id : partition_ids) { executor.add([&, partition_id, this]() { - return commit_partition(db_id, partition_id, - partition_to_tmp_rowset_metas.at(partition_id), - is_versioned_read, is_versioned_write, - defer_deleting_pending_delete_bitmaps); + return commit_partition( + db_id, partition_id, partition_to_tmp_rowset_metas.at(partition_id), + is_versioned_read, is_versioned_write, + defer_deleting_pending_delete_bitmaps, txn_info.commit_tso()); }); } bool finished = false; @@ -758,7 +760,7 @@ void TxnLazyCommitTask::commit() { std::tie(code_, msg_) = commit_partition( db_id, partition_id, partition_to_tmp_rowset_metas[partition_id], is_versioned_read, is_versioned_write, - defer_deleting_pending_delete_bitmaps); + defer_deleting_pending_delete_bitmaps, txn_info.commit_tso()); if (code_ != MetaServiceCode::OK) break; } } @@ -777,8 +779,8 @@ void TxnLazyCommitTask::commit() { std::pair TxnLazyCommitTask::commit_partition( int64_t db_id, int64_t partition_id, const std::vector>& tmp_rowset_metas, - bool is_versioned_read, bool is_versioned_write, - bool defer_deleting_pending_delete_bitmaps) { + bool is_versioned_read, bool is_versioned_write, bool defer_deleting_pending_delete_bitmaps, + int64_t commit_tso) { std::stringstream ss; CloneChainReader meta_reader(instance_id_, txn_kv_.get(), txn_lazy_committer_->resource_manager().get()); @@ -852,7 +854,7 @@ std::pair TxnLazyCommitTask::commit_partition( sub_partition_tmp_rowset_metas, tablet_ids, is_versioned_write, is_versioned_read, versionstamp, txn_lazy_committer_->resource_manager().get(), - defer_deleting_pending_delete_bitmaps); + defer_deleting_pending_delete_bitmaps, commit_tso); if (code != MetaServiceCode::OK) { return {code, msg}; } diff --git a/cloud/src/meta-service/txn_lazy_committer.h b/cloud/src/meta-service/txn_lazy_committer.h index 0ac6591029c60c..94d36625587776 100644 --- a/cloud/src/meta-service/txn_lazy_committer.h +++ b/cloud/src/meta-service/txn_lazy_committer.h @@ -54,7 +54,7 @@ class TxnLazyCommitTask { int64_t db_id, int64_t partition_id, const std::vector>& tmp_rowset_metas, bool is_versioned_write, bool is_versioned_read, - bool defer_deleting_pending_delete_bitmaps); + bool defer_deleting_pending_delete_bitmaps, int64_t commit_tso); std::string instance_id_; int64_t txn_id_; diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index d313467b127287..93c0608a2820fe 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -670,10 +670,6 @@ private static void initClusterGuard(String dorisHomeDir) throws Exception { } public static void overwriteConfigs() { - if (Config.isCloudMode() && Config.enable_feature_binlog) { - Config.enable_feature_binlog = false; - LOG.warn("Force set enable_feature_binlog=false because it is not supported in the cloud mode yet"); - } } private static void fuzzyConfigs() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java index 3de8b2c6659961..5feb876cb47ed2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java @@ -113,7 +113,9 @@ public boolean checkBinlogConfigChange(List alterOps) { ).anyMatch(clause -> clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_ENABLE) || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_TTL_SECONDS) || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_MAX_BYTES) - || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_MAX_HISTORY_NUMS)); + || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_MAX_HISTORY_NUMS) + || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_FORMAT) + || clause.getProperties().containsKey(PropertyAnalyzer.PROPERTIES_BINLOG_NEED_HISTORICAL_VALUE)); } public boolean isBeingSynced(List alterOps) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index e3192f5e4911b3..a3ecde58ee06f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -233,7 +233,7 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), - tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(), + tbl.getBaseSchemaVersion(), null, tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -247,7 +247,7 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), tbl.storageDictPageSize(), true, tbl.getColumnSeqMapping(), - tbl.getVerticalCompactionNumColumnsPerGroup()); + tbl.getVerticalCompactionNumColumnsPerGroup(), false); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index cd1b90fb923316..4028c467d9cc61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -255,7 +255,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), - shadowSchemaVersion, tbl.getCompactionPolicy(), + shadowSchemaVersion, null, tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -269,7 +269,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), tbl.storageDictPageSize(), true, columnSeqMapping, - tbl.getVerticalCompactionNumColumnsPerGroup()); + tbl.getVerticalCompactionNumColumnsPerGroup(), false); requestBuilder.addTabletMetas(builder); } // end for rollupTablets requestBuilder.setDbId(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 8dd9d6ba1a20df..3ab2ece99dae42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -272,7 +272,7 @@ protected void createRollupReplica() throws AlterCancelException { tbl.variantEnableFlattenNested(), tbl.storagePageSize(), tbl.getTDEAlgorithm(), tbl.storageDictPageSize(), null, - tbl.getVerticalCompactionNumColumnsPerGroup(), null); + tbl.getVerticalCompactionNumColumnsPerGroup()); createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash); if (this.storageFormat != null) { createReplicaTask.setStorageFormat(this.storageFormat); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 233cead1bdce43..8dc4041ec0d820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -3949,6 +3949,17 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List for table: " + + olapTable.getName()); + } + + if (newBinlogConfig.isEnableForCCR()) { + if (Config.isCloudMode()) { + throw new DdlException("Binlog is not supported in cloud mode"); + } + } + // check db binlog config, if db binlog config is not same as table binlog config, throw exception BinlogConfig dbBinlogConfig; db.readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index c044a8c258f804..976577f7ec8044 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -308,11 +308,6 @@ protected void createShadowIndexReplica() throws AlterCancelException { long shadowReplicaId = shadowReplica.getId(); countDownLatch.addMark(backendId, shadowTabletId); - MaterializedIndexMeta rowBinlogIndexMeta = null; - if (tbl.needRowBinlog() && originIndexId == tbl.getBaseIndexId()) { - rowBinlogIndexMeta = tbl.getRowBinlogMeta(); - } - CreateReplicaTask createReplicaTask = new CreateReplicaTask( backendId, dbId, tableId, partitionId, shadowIdxId, shadowTabletId, shadowReplicaId, shadowShortKeyColumnCount, shadowSchemaHash, @@ -341,8 +336,7 @@ protected void createShadowIndexReplica() throws AlterCancelException { tbl.storagePageSize(), tbl.getTDEAlgorithm(), tbl.storageDictPageSize(), columnSeqMapping, - tbl.getVerticalCompactionNumColumnsPerGroup(), - rowBinlogIndexMeta); + tbl.getVerticalCompactionNumColumnsPerGroup()); createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId) .get(shadowTabletId), originSchemaHash); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index f64544415536be..2bb8a19efa4ed6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -704,6 +704,12 @@ private void checkAndPrepareMeta() { BackupOlapTableInfo tblInfo = olapTableEntry.getValue(); Table remoteTbl = backupMeta.getTable(tableName); Preconditions.checkNotNull(remoteTbl); + OlapTable remoteOlapTbl = (OlapTable) remoteTbl; + if (remoteOlapTbl.needRowBinlog()) { + status = new Status(ErrCode.COMMON_ERROR, + "Do not support restore table with binlog enabled: " + remoteOlapTbl.getName()); + return; + } Table localTbl = db.getTableNullable(jobInfo.getAliasByOriginNameIfSet(tableName)); boolean isSchemaChanged = false; if (localTbl != null && localTbl.getType() != TableType.OLAP) { @@ -722,8 +728,6 @@ private void checkAndPrepareMeta() { + localOlapTbl.getName()); return; } - OlapTable remoteOlapTbl = (OlapTable) remoteTbl; - if (localOlapTbl.isColocateTable() || (reserveColocate && remoteOlapTbl.isColocateTable())) { status = new Status(ErrCode.COMMON_ERROR, "Not support to restore to local table " + tableName + " with colocate group."); @@ -870,7 +874,6 @@ private void checkAndPrepareMeta() { // Table does not exist or atomic restore if (localTbl == null || isAtomicRestore) { - OlapTable remoteOlapTbl = (OlapTable) remoteTbl; // Retain only expected restore partitions in this table; Set allPartNames = remoteOlapTbl.getPartitionNames(); for (String partName : allPartNames) { @@ -1458,10 +1461,6 @@ protected void createReplicas(Database db, OlapTable localTbl, Partition restore Env.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); for (Replica restoreReplica : restoreTablet.getReplicas()) { Env.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); - MaterializedIndexMeta rowBinlogIndexMeta = null; - if (localTbl.needRowBinlog() && restoredIdx.getId() == localTbl.getBaseIndexId()) { - rowBinlogIndexMeta = localTbl.getRowBinlogMeta(); - } CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendIdWithoutException(), dbId, localTbl.getId(), restorePart.getId(), restoredIdx.getId(), restoreTablet.getId(), restoreReplica.getId(), indexMeta.getShortKeyColumnCount(), @@ -1492,8 +1491,7 @@ protected void createReplicas(Database db, OlapTable localTbl, Partition restore localTbl.storagePageSize(), localTbl.getTDEAlgorithm(), localTbl.storageDictPageSize(), localTbl.getColumnSeqMapping(), - localTbl.getVerticalCompactionNumColumnsPerGroup(), - rowBinlogIndexMeta); + localTbl.getVerticalCompactionNumColumnsPerGroup()); task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat()); task.setInRestoreMode(true); if (baseTabletRef != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BinlogConfig.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BinlogConfig.java index 56323c1c4faea2..f36c3c7058f8ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BinlogConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BinlogConfig.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.proto.OlapFile; import org.apache.doris.thrift.TBinlogConfig; import org.apache.doris.thrift.TBinlogFormat; @@ -220,6 +221,19 @@ public TBinlogConfig toThrift() { return tBinlogConfig; } + public OlapFile.BinlogConfigPB toProtobuf() { + OlapFile.BinlogConfigPB.Builder binlogConfigBuilder = OlapFile.BinlogConfigPB.newBuilder(); + binlogConfigBuilder.setEnable(enable); + binlogConfigBuilder.setTtlSeconds(ttlSeconds); + binlogConfigBuilder.setMaxBytes(maxBytes); + binlogConfigBuilder.setMaxHistoryNums(maxHistoryNums); + if (binlogFormat != null) { + binlogConfigBuilder.setBinlogFormat(OlapFile.BinlogFormatPB.valueOf(binlogFormat.name())); + } + binlogConfigBuilder.setNeedHistoricalValue(needHistoricalValue); + return binlogConfigBuilder.build(); + } + public Map toProperties() { Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_BINLOG_ENABLE, String.valueOf(enable)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 9ad1594bc4c0a2..0fd215ee076885 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -1024,7 +1024,7 @@ public void recoverPartition(long dbId, OlapTable table, String partitionName, // check if schema change Partition recoverPartition = recoverPartitionInfo.getPartition(); - Set tableIndex = table.getIndexIdToMeta().keySet(); + Set tableIndex = table.getIndexIdToMetaWithRowBinlog().keySet(); Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() .map(i -> i.getId()).collect(Collectors.toSet()); if (!tableIndex.equals(partitionIndex)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java index 118aa401227849..df0c5b6bf4da22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java @@ -182,7 +182,8 @@ private List getAllTabletStats(Function filter) { try { OlapTable tbl = (OlapTable) table; for (Partition partition : tbl.getAllPartitions()) { - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { for (Tablet tablet : index.getTablets()) { if (filter != null && !filter.apply((CloudTablet) tablet)) { continue; @@ -353,14 +354,18 @@ private void updateStatInfo(List dbIds) { dynamicPartitionNearLimitCount++; } } + long tableBinlogSize = 0L; + long tableTotalBinlogSize = 0L; for (Partition partition : allPartitions) { long partitionDataSize = 0L; - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { long indexRowCount = 0L; List tablets = index.getTablets(); tabletCount += tablets.size(); for (Tablet tablet : tablets) { long tabletDataSize = 0L; + long tabletBinlogSize = 0L; long tabletRowCount = 0L; long tabletIndexSize = 0L; @@ -373,6 +378,13 @@ private void updateStatInfo(List dbIds) { tableTotalReplicaDataSize += replica.getDataSize(); } + if (index.isRowBinlog()) { + if (replica.getDataSize() > tabletBinlogSize) { + tabletBinlogSize = replica.getDataSize(); + } + tableTotalBinlogSize += replica.getDataSize(); + } + if (replica.getRowCount() > tabletRowCount) { tabletRowCount = replica.getRowCount(); } @@ -389,6 +401,7 @@ private void updateStatInfo(List dbIds) { tableDataSize += tabletDataSize; partitionDataSize += tabletDataSize; + tableBinlogSize += tabletBinlogSize; if (maxTabletSize.second <= tabletDataSize) { maxTabletSize = Pair.of("" + tablet.getId(), tabletDataSize); } @@ -396,8 +409,10 @@ private void updateStatInfo(List dbIds) { minTabletSize = Pair.of("" + tablet.getId(), tabletDataSize); } - tableRowCount += tabletRowCount; indexRowCount += tabletRowCount; + if (!index.isRowBinlog()) { + tableRowCount += tabletRowCount; + } tableTotalLocalIndexSize += tabletIndexSize; tableTotalLocalSegmentSize += tabletSegmentSize; @@ -423,7 +438,8 @@ private void updateStatInfo(List dbIds) { tableStats = new OlapTable.Statistics(db.getName(), table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L, tableReplicaCount, tableRowCount, 0L, 0L, - tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L, 0L, 0L); + tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L, + tableBinlogSize, tableTotalBinlogSize); olapTable.setStatistics(tableStats); LOG.debug("finished to set row num for table: {} in database: {}", table.getName(), db.getFullName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index f6f21be4977633..1f5516da3c1701 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -1017,6 +1017,12 @@ public boolean updateDbProperties(Map properties) throws DdlExce BinlogConfig oldBinlogConfig = getBinlogConfig(); BinlogConfig newBinlogConfig = BinlogConfig.fromProperties(properties); + if (newBinlogConfig.isEnableForCCR()) { + if (Config.isCloudMode()) { + throw new DdlException("Binlog is not supported in cloud mode"); + } + } + if (newBinlogConfig.isEnableForCCR() && !oldBinlogConfig.isEnableForCCR()) { // check all tables binlog enable is true for (Table table : idToTable.values()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java index 65d6a295769190..8cef574c204db2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java @@ -365,11 +365,6 @@ private void updateReplicaBasicInfo(Replica replica, TTabletInfo backendTabletIn && replica.getSchemaHash() != backendTabletInfo.getSchemaHash()) { replica.setSchemaHash(backendTabletInfo.getSchemaHash()); } - - if (backendTabletInfo.isSetBinlogSize()) { - replica.setBinlogSize(backendTabletInfo.getBinlogSize()); - replica.setBinlogFileNum(backendTabletInfo.getBinlogFileNum()); - } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 74b36e31d46637..373f343f16560c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -34,21 +34,33 @@ public class MaterializedIndex extends MetaObject implements GsonPostProcessable { public enum IndexState { NORMAL, + ROW_BINLOG, @Deprecated ROLLUP, @Deprecated SCHEMA_CHANGE, - SHADOW; // index in SHADOW state is visible to load process, but invisible to query + SHADOW, // index in SHADOW state is visible to load process, but invisible to query + SHADOW_ROW_BINLOG; public boolean isVisible() { - return this == IndexState.NORMAL; + return this == IndexState.NORMAL || this == IndexState.ROW_BINLOG; + } + + public boolean isRowBinlog() { + return this == IndexState.ROW_BINLOG || this == IndexState.SHADOW_ROW_BINLOG; + } + + public boolean isShadow() { + return this == IndexState.SHADOW || this == IndexState.SHADOW_ROW_BINLOG; } } public enum IndexExtState { ALL, - VISIBLE, // index state in NORMAL - SHADOW // index state in SHADOW + ALL_EXCEPT_ROW_BINLOG, // exclude row binlog + VISIBLE, // exclude row binlog + VISIBLE_WITH_ROW_BINLOG, + SHADOW // exclude row binlog } @SerializedName(value = "id") @@ -100,6 +112,10 @@ public MaterializedIndex(long id, IndexState state) { this.rollupFinishedVersion = -1L; } + public boolean isRowBinlog() { + return state.isRowBinlog(); + } + public List getTablets() { // Volatile read: returns the current immutable snapshot; callers iterate without locking. return Collections.unmodifiableList(tablets); @@ -223,11 +239,7 @@ public long getRemoteDataSize() { } public long getBinlogSize() { - long binlogDataSize = 0; - for (Tablet tablet : getTablets()) { - binlogDataSize += tablet.getBinlogDataSize(); - } - return binlogDataSize; + return isRowBinlog() ? getDataSize(false, false) : 0; } public long getReplicaCount() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java index ce9524343a3c7c..8494faf4cf7759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java @@ -80,7 +80,8 @@ private static List> getTabletStatus(String dbName, String tblName, short replicationNum = olapTable.getPartitionInfo() .getReplicaAllocation(partition.getId()).getTotalReplicaNum(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getMaterializedIndices( + IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -187,7 +188,8 @@ private static List> getTabletStatus(String dbName, String tblName, short replicationNum = olapTable.getPartitionInfo() .getReplicaAllocation(partition.getId()).getTotalReplicaNum(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getMaterializedIndices( + IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -343,7 +345,8 @@ public static List> getTabletDistribution( long totalReplicaSize = 0; for (long partId : partitionIds) { Partition partition = olapTable.getPartition(partId); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getMaterializedIndices( + IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { long beId = replica.getBackendIdWithoutException(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..43256ee4fa2e98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -722,11 +722,17 @@ private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List names, L } public List getVisibleIndex() { + return getVisibleIndexWithRowBinlog().stream() + .filter(index -> !index.isRowBinlog()) + .collect(Collectors.toList()); + } + + public List getVisibleIndexWithRowBinlog() { Optional partition = idToPartition.values().stream().findFirst(); if (!partition.isPresent()) { partition = tempPartitions.getAllPartitions().stream().findFirst(); } - return partition.isPresent() ? partition.get().getMaterializedIndices(IndexExtState.VISIBLE) + return partition.isPresent() ? partition.get().getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG) : Collections.emptyList(); } @@ -1082,6 +1088,10 @@ public int getIndexNumber() { return getIndexIdToMeta().size(); } + public int getIndexNumberWithRowBinlog() { + return indexIdToMeta.size(); + } + public Map getIndexIdToMeta() { return ImmutableMap.copyOf(Maps.filterValues(indexIdToMeta, meta -> !meta.isRowBinlogIndex())); } @@ -1098,6 +1108,17 @@ public Map getCopiedIndexIdToMeta() { return new HashMap<>(getIndexIdToMeta()); } + // Includes the hidden row binlog index meta (which getIndexIdToMeta filters out). + // Used by partition creation paths that need to create the companion binlog tablet. + // For tables without row binlog this equals getIndexIdToMeta. + public Map getIndexIdToMetaWithRowBinlog() { + return ImmutableMap.copyOf(indexIdToMeta); + } + + public Map getCopiedIndexIdToMetaWithRowBinlog() { + return new HashMap<>(indexIdToMeta); + } + public MaterializedIndexMeta getIndexMetaByIndexId(long indexId) { return indexIdToMeta.get(indexId); } @@ -1112,6 +1133,16 @@ public List getIndexIdListExceptBaseIndex() { return result; } + public List getIndexIdListWithRowBinlogExceptBaseIndex() { + List result = Lists.newArrayList(); + for (Long indexId : getIndexIdToMetaWithRowBinlog().keySet()) { + if (indexId != baseIndexId) { + result.add(indexId); + } + } + return result; + } + public List getIndexIdList() { List result = Lists.newArrayList(); for (Long indexId : getIndexIdToMeta().keySet()) { @@ -1120,6 +1151,10 @@ public List getIndexIdList() { return result; } + public List getIndexIdListWithRowBinlog() { + return Lists.newArrayList(indexIdToMeta.keySet()); + } + // schema public Map> getIndexIdToSchema() { return getIndexIdToSchema(Util.showHiddenColumns()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java index d9e9885562fd19..832b8f0821df7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java @@ -261,11 +261,13 @@ public List getMaterializedIndices(IndexExtState extState) { List indices = Lists.newArrayList(); switch (extState) { case ALL: + case ALL_EXCEPT_ROW_BINLOG: indices.add(baseIndex); indices.addAll(idToVisibleRollupIndex.values()); indices.addAll(idToShadowIndex.values()); break; case VISIBLE: + case VISIBLE_WITH_ROW_BINLOG: indices.add(baseIndex); indices.addAll(idToVisibleRollupIndex.values()); break; @@ -275,6 +277,9 @@ public List getMaterializedIndices(IndexExtState extState) { default: break; } + if (extState != IndexExtState.ALL && extState != IndexExtState.VISIBLE_WITH_ROW_BINLOG) { + indices.removeIf(MaterializedIndex::isRowBinlog); + } return indices; } @@ -297,7 +302,7 @@ public String getMetaChecksum() { } else { updateMetaChecksum(digest, (byte) 17, -1L); } - List indexes = getMaterializedIndices(IndexExtState.VISIBLE); + List indexes = getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG); indexes.sort(Comparator.comparingLong(MaterializedIndex::getId)); for (MaterializedIndex index : indexes) { updateMetaChecksum(digest, (byte) 1, index.getId()); @@ -364,7 +369,7 @@ public long getAllDataSize(boolean singleReplica) { // this is local data size public long getDataSize(boolean singleReplica) { long dataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { dataSize += mIndex.getDataSize(singleReplica, false); } return dataSize; @@ -372,7 +377,7 @@ public long getDataSize(boolean singleReplica) { public long getRemoteDataSize() { long remoteDataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { remoteDataSize += mIndex.getRemoteDataSize(); } return remoteDataSize; @@ -380,7 +385,7 @@ public long getRemoteDataSize() { public long getBinlogDataSize() { long binlogDataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { binlogDataSize += mIndex.getBinlogSize(); } return binlogDataSize; @@ -388,7 +393,7 @@ public long getBinlogDataSize() { public long getReplicaCount() { long replicaCount = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { replicaCount += mIndex.getReplicaCount(); } return replicaCount; @@ -508,7 +513,7 @@ public long getDataLength() { public long getDataSizeExcludeEmptyReplica(boolean singleReplica) { long dataSize = 0; - for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { dataSize += mIndex.getDataSize(singleReplica, true); } return dataSize + getRemoteDataSize(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index eb5dc548c504e0..06e1e8ec42cac9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -27,7 +27,6 @@ import org.apache.doris.resource.Tag; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TTabletCopyType; import com.google.common.base.Joiner; import com.google.common.collect.HashMultimap; @@ -104,6 +103,10 @@ public TabletHealth() { @SerializedName(value = "id") protected long id; + // The aligned tablet for the base <-> row binlog tablet relationship: + // on a base tablet it holds the binlog tablet id, on a binlog tablet it holds the base tablet id. + @SerializedName(value = "ati") + protected long alignedTabletId = -1; public Tablet() { this(0L); @@ -117,6 +120,14 @@ public long getId() { return this.id; } + public long getAlignedTabletId() { + return alignedTabletId; + } + + public void setAlignedTabletId(long alignedTabletId) { + this.alignedTabletId = alignedTabletId; + } + public long getCheckedVersion() { return -1; } @@ -344,16 +355,6 @@ public long getRemoteDataSize() { return 0; } - public long getBinlogDataSize() { - long binlogDataSize = 0; - for (Replica replica : getReplicas()) { - if (replica.getState() == ReplicaState.NORMAL) { - binlogDataSize += replica.getBinlogSize(); - } - } - return binlogDataSize; - } - public abstract long getRowCount(boolean singleReplica); // Get the least row count among all valid replicas. @@ -731,26 +732,4 @@ public long getLastCheckTime() { public void setLastCheckTime(long lastCheckTime) { throw new UnsupportedOperationException("setLastCheckTime is not supported in Tablet"); } - - public static class CopyType { - public static final int DEFAULT = TTabletCopyType.DATA.getValue() - | TTabletCopyType.CCR_BINLOG.getValue(); - - public static boolean has(int copyType, TTabletCopyType type) { - return (copyType & type.getValue()) != 0; - } - - public static void validate(int copyType) { - if (copyType <= 0 || (copyType & ~allTypes()) != 0) { - throw new IllegalArgumentException( - "invalid copy_type bitmask: " + copyType + ", valid bits: " + allTypes()); - } - } - - private static int allTypes() { - return TTabletCopyType.DATA.getValue() - | TTabletCopyType.ROW_BINLOG.getValue() - | TTabletCopyType.CCR_BINLOG.getValue(); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index ec99c8d45a991a..bb5ba53d270d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -188,7 +188,8 @@ protected void runAfterCatalogReady() { for (Partition partition : allPartitions) { long partitionDataSize = 0L; long version = partition.getVisibleVersion(); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { long indexRowCount = 0L; boolean indexReported = true; List tablets = index.getTablets(); @@ -242,10 +243,14 @@ protected void runAfterCatalogReady() { tableTotalRemoteIndexSize += replica.getRemoteInvertedIndexSize(); tableTotalRemoteSegmentSize += replica.getRemoteSegmentSize(); - if (replica.getBinlogSize() > tabletBinlogSize) { - tabletBinlogSize = replica.getBinlogSize(); + // Binlog is now an independent index. Its data is counted in table data size + // for quota, and also tracked separately as binlog size. + if (index.isRowBinlog()) { + if (replica.getDataSize() > tabletBinlogSize) { + tabletBinlogSize = replica.getDataSize(); + } + tableTotalBinlogSize += replica.getDataSize(); } - tableTotalBinlogSize += replica.getBinlogSize(); } tableDataSize += tabletDataSize; @@ -262,8 +267,10 @@ protected void runAfterCatalogReady() { if (tabletRowCount == Long.MAX_VALUE) { tabletRowCount = 0L; } - tableRowCount += tabletRowCount; indexRowCount += tabletRowCount; + if (!index.isRowBinlog()) { + tableRowCount += tabletRowCount; + } // Only when all tablets of this index are reported, we set indexReported to true. indexReported = indexReported && tabletReported; @@ -404,11 +411,6 @@ private void updateTabletStat(Long beId, TTabletStatResult result) { // Older version BE doesn't set visible version. Set it to max for compatibility. replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion() : Long.MAX_VALUE); - - if (stat.isSetBinlogSize()) { - replica.setBinlogSize(stat.getBinlogSize()); - replica.setBinlogFileNum(stat.getBinlogFileNum()); - } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java index 7dc0a9125dddf7..3b38966291c19a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java @@ -521,7 +521,8 @@ private void matchGroups() { long visibleVersion = partition.getVisibleVersion(); // Here we only get VISIBLE indexes. All other indexes are not queryable. // So it does not matter if tablets of other indexes are not matched. - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { Preconditions.checkState(backendBucketsSeq.size() == index.getTablets().size(), backendBucketsSeq.size() + " vs. " + index.getTablets().size()); List tabletIdsInOrder = index.getTabletIdsInOrder(); @@ -644,7 +645,8 @@ private GlobalColocateStatistic buildGlobalColocateStatistic() { // Here we only get VISIBLE indexes. All other indexes are not queryable. // So it does not matter if tablets of other indexes are not matched. - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { Preconditions.checkState(backendBucketsSeq.size() == index.getTablets().size(), backendBucketsSeq.size() + " vs. " + index.getTablets().size()); int tabletOrderIdx = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java index e652fc68d460ba..5c7b19196f5b64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java @@ -366,7 +366,7 @@ private LoopControlStatus handlePartitionTablet(Database db, OlapTable tbl, Part /* * Tablet in SHADOW index can not be repaired of balanced */ - for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex idx : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { for (Tablet tablet : idx.getTablets()) { counter.totalTabletNum++; diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 8665dd69d86324..bc1809d1a511ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -28,7 +28,6 @@ import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; -import org.apache.doris.catalog.Tablet.CopyType; import org.apache.doris.catalog.Tablet.TabletHealth; import org.apache.doris.catalog.Tablet.TabletStatus; import org.apache.doris.clone.SchedException.Status; @@ -50,7 +49,6 @@ import org.apache.doris.thrift.TFinishTaskRequest; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageMedium; -import org.apache.doris.thrift.TTabletCopyType; import org.apache.doris.thrift.TTabletInfo; import org.apache.doris.thrift.TTaskType; @@ -161,7 +159,6 @@ public enum State { private Tablet tablet = null; private long visibleVersion = -1; private long committedVersion = -1; - private boolean copyRowBinlog = false; private long tabletSize = 0; @@ -394,10 +391,6 @@ public void setTablet(Tablet tablet) { this.tablet = tablet; } - public void setCopyRowBinlog(boolean copyRowBinlog) { - this.copyRowBinlog = copyRowBinlog; - } - public Tablet getTablet() { return tablet; } @@ -1056,11 +1049,6 @@ public CloneTask createCloneReplicaAndTask() throws SchedException { cloneTask = new CloneTask(tDestBe, destBackendId, dbId, tblId, partitionId, indexId, tabletId, replica.getId(), schemaHash, Lists.newArrayList(tSrcBe), storageMedium, visibleVersion, (int) (taskTimeoutMs / 1000)); - int copyType = CopyType.DEFAULT; - if (copyRowBinlog) { - copyType |= TTabletCopyType.ROW_BINLOG.getValue(); - } - cloneTask.setCopyType(copyType); destOldVersion = replica.getVersion(); cloneTask.setPathHash(srcPathHash, destPathHash); LOG.info("create clone task to repair replica, tabletId={}, replica={}, visible version {}, tablet status {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index acf40add785249..a15830efe0151b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -645,7 +645,6 @@ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) tabletCtx.updateTabletSize(); tabletCtx.setVersionInfo(partition.getVisibleVersion(), partition.getCommittedVersion()); tabletCtx.setSchemaHash(tbl.getSchemaHashByIndexId(idx.getId())); - tabletCtx.setCopyRowBinlog(idx.getId() == tbl.getBaseIndexId() && tbl.needRowBinlog()); tabletCtx.setStorageMedium(tbl.getPartitionInfo().getDataProperty(partition.getId()).getStorageMedium()); handleTabletByTypeAndStatus(tabletHealth.status, tabletCtx, batchTask); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java index aafb0c667164db..45602e8d3ecae9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java @@ -427,8 +427,10 @@ public void createReplicas(Database db, OlapTable localTbl, Partition restorePar localTbl.getCompressionType(), localTbl.getStorageFormat(), localTbl.getStoragePolicy(), localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(), - localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(), - localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(), + !indexMeta.isRowBinlogIndex() && localTbl.getEnableUniqueKeyMergeOnWrite(), + localTbl.storeRowColumn(), + localTbl.getBaseSchemaVersion(), localTbl.getBinlogConfig(), + localTbl.getCompactionPolicy(), localTbl.getTimeSeriesCompactionGoalSizeMbytes(), localTbl.getTimeSeriesCompactionFileCountThreshold(), localTbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -441,7 +443,8 @@ public void createReplicas(Database db, OlapTable localTbl, Partition restorePar localTbl.storagePageSize(), localTbl.getTDEAlgorithmPB(), localTbl.storageDictPageSize(), false, localTbl.getColumnSeqMapping(), - localTbl.getVerticalCompactionNumColumnsPerGroup())); + localTbl.getVerticalCompactionNumColumnsPerGroup(), + indexMeta.isRowBinlogIndex())); // In cloud mode all storage medium will be saved to HDD. TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ee3091ec8e1751..4868ac70808b34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -59,6 +59,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.util.ColumnsUtil; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.filesystem.spi.RemoteObject; import org.apache.doris.proto.OlapCommon; @@ -128,7 +129,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa distributionInfo, dbId, tbl.getId()); // add to index map - Map indexMap = Maps.newHashMap(); + // Use LinkedHashMap so the base index is processed before the row-binlog companion index. + Map indexMap = Maps.newLinkedHashMap(); indexMap.put(tbl.getBaseIndexId(), baseIndex); // create rollup index if has @@ -137,7 +139,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa continue; } - MaterializedIndex rollup = new MaterializedIndex(indexId, IndexState.NORMAL); + MaterializedIndex rollup = new MaterializedIndex(indexId, + indexIdToMeta.get(indexId).isRowBinlogIndex() ? IndexState.ROW_BINLOG : IndexState.NORMAL); indexMap.put(indexId, rollup); } @@ -148,18 +151,24 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa long indexId = entry.getKey(); MaterializedIndex index = entry.getValue(); MaterializedIndexMeta indexMeta = indexIdToMeta.get(indexId); + boolean isRowBinlogIndex = indexMeta.isRowBinlogIndex(); // create tablets int schemaHash = indexMeta.getSchemaHash(); TabletMeta tabletMeta = new TabletMeta(dbId, tbl.getId(), partitionId, indexId, schemaHash, dataProperty.getStorageMedium()); - createCloudTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, - tabletMeta, tabletIdSet); + if (isRowBinlogIndex) { + createCloudRowBinlogTablets(index, baseIndex, version, tabletMeta, tabletIdSet); + } else { + createCloudTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, + tabletMeta, tabletIdSet); + } short shortKeyColumnCount = indexMeta.getShortKeyColumnCount(); // TStorageType storageType = indexMeta.getStorageType(); List columns = indexMeta.getSchema(); KeysType keysType = indexMeta.getKeysType(); + boolean enableUniqueKeyMergeOnWrite = !isRowBinlogIndex && tbl.getEnableUniqueKeyMergeOnWrite(); List indexes; if (index.getId() == tbl.getBaseIndexId()) { @@ -182,8 +191,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(), - tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(), - tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), + enableUniqueKeyMergeOnWrite, tbl.storeRowColumn(), indexMeta.getSchemaVersion(), + binlogConfig, tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), @@ -196,7 +205,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), tbl.storageDictPageSize(), true, tbl.getColumnSeqMapping(), - tbl.getVerticalCompactionNumColumnsPerGroup()); + tbl.getVerticalCompactionNumColumnsPerGroup(), isRowBinlogIndex); requestBuilder.addTabletMetas(builder); } requestBuilder.setDbId(dbId); @@ -222,7 +231,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, TStorageFormat storageFormat, String storagePolicy, boolean isInMemory, boolean isShadow, String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, - boolean storeRowColumn, int schemaVersion, String compactionPolicy, + boolean storeRowColumn, int schemaVersion, BinlogConfig binlogConfig, String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, @@ -231,7 +240,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, boolean variantEnableFlattenNested, List clusterKeyUids, long storagePageSize, EncryptionAlgorithmPB encryptionAlgorithm, long storageDictPageSize, boolean createInitialRowset, Map> columnSeqMapping, - int verticalCompactionNumColumnsPerGroup) throws DdlException { + int verticalCompactionNumColumnsPerGroup, boolean isRowBinlogTablet) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -245,6 +254,10 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, builder.setIsInMemory(isInMemory); builder.setTtlSeconds(ttlSeconds); builder.setSchemaVersion(schemaVersion); + builder.setIsRowBinlogTablet(isRowBinlogTablet); + if (binlogConfig != null) { + builder.setBinlogConfig(binlogConfig.toProtobuf()); + } UUID uuid = UUID.randomUUID(); Types.PUniqueId tabletUid = Types.PUniqueId.newBuilder() @@ -260,7 +273,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, builder.setReplicaId(tablet.getReplicas().get(0).getId()); builder.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); - builder.setCompactionPolicy(compactionPolicy); + builder.setCompactionPolicy(isRowBinlogTablet + ? PropertyAnalyzer.BINLOG_COMPACTION_POLICY : compactionPolicy); builder.setTimeSeriesCompactionGoalSizeMbytes(timeSeriesCompactionGoalSizeMbytes); builder.setTimeSeriesCompactionFileCountThreshold(timeSeriesCompactionFileCountThreshold); builder.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); @@ -481,6 +495,29 @@ private void createCloudTablets(MaterializedIndex index, ReplicaState replicaSta index.appendTablets(bucketTablets); } + private void createCloudRowBinlogTablets(MaterializedIndex rowBinlogIndex, MaterializedIndex baseIndex, + long version, TabletMeta tabletMeta, Set tabletIdSet) { + TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex(); + List baseTablets = baseIndex.getTablets(); + List rowBinlogTablets = new ArrayList<>(baseTablets.size()); + for (int i = 0; i < baseTablets.size(); ++i) { + Tablet baseTablet = baseTablets.get(i); + Tablet rowBinlogTablet = EnvFactory.getInstance().createTablet(Env.getCurrentEnv().getNextId()); + baseTablet.setAlignedTabletId(rowBinlogTablet.getId()); + rowBinlogTablet.setAlignedTabletId(baseTablet.getId()); + invertedIndex.addTablet(rowBinlogTablet.getId(), tabletMeta); + rowBinlogTablets.add(rowBinlogTablet); + tabletIdSet.add(rowBinlogTablet.getId()); + + long replicaId = Env.getCurrentEnv().getNextId(); + Replica replica = new CloudReplica(replicaId, null, ReplicaState.NORMAL, version, + tabletMeta.getOldSchemaHash(), tabletMeta.getDbId(), tabletMeta.getTableId(), + tabletMeta.getPartitionId(), tabletMeta.getIndexId(), i); + rowBinlogTablet.addReplica(replica); + } + rowBinlogIndex.appendTablets(rowBinlogTablets); + } + @Override public void beforeCreatePartitions(long dbId, long tableId, List partitionIds, List indexIds, boolean isCreateTable) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 1cf69114d0981e..d103ba582a8e0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -436,6 +436,9 @@ public void commitTransactionWithoutLock(long dbId, List tableList, long LOG.info("try to commit transaction, transactionId: {}, tableIds: {}", transactionId, tableList.stream().map(Table::getId).collect(Collectors.toList())); Map> backendToPartitionInfos = null; + Database database = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId); + long commitTSO = TransactionUtil.getCommitTSO(transactionId, database, + tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(dbId, transactionId)) { return; @@ -449,7 +452,7 @@ public void commitTransactionWithoutLock(long dbId, List
tableList, long backendToPartitionInfos = getCalcDeleteBitmapInfo(lockContext, null); } commitTransactionWithoutLock(dbId, tableList, transactionId, tabletCommitInfos, txnCommitAttachment, false, - mowTableList, backendToPartitionInfos); + mowTableList, backendToPartitionInfos, commitTSO); // clear signature after commit succeeds clearTxnLastSignature(dbId, transactionId); } catch (Exception e) { @@ -683,7 +686,8 @@ private Set getBaseTabletsFromTables(List
tableList, List tableList, long transactionId, List tabletCommitInfos, TxnCommitAttachment txnCommitAttachment, boolean is2PC, - List mowTableList, Map> backendToPartitionInfos) + List mowTableList, Map> backendToPartitionInfos, + long commitTSO) throws UserException { if (Config.disable_load_job) { throw new TransactionCommitFailedException( @@ -703,6 +707,7 @@ private void commitTransactionWithoutLock(long dbId, List
tableList, long .setTxnId(transactionId) .setIs2Pc(is2PC) .setCloudUniqueId(Config.cloud_unique_id) + .setCommitTso(commitTSO) .addAllBaseTabletIds(getBaseTabletsFromTables(tableList, tabletCommitInfos)) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { @@ -1019,6 +1024,19 @@ private void getPartitionInfo(List tableList, if (!tableMap.containsKey(tableId)) { continue; } + long partitionId = tabletMeta.getPartitionId(); + Partition partition = tableMap.get(tableId).getPartition(partitionId); + if (partition == null) { + throw new MetaNotFoundException("partition " + partitionId + " does not exist"); + } + MaterializedIndex index = partition.getIndex(tabletMeta.getIndexId()); + if (index == null) { + throw new MetaNotFoundException("index " + tabletMeta.getIndexId() + " does not exist"); + } + if (index.isRowBinlog()) { + // Row-binlog delete bitmap is derived and saved by its base tablet. + continue; + } lockContext.getTabletToTabletMeta().put(tabletId, tabletMeta); @@ -1028,7 +1046,6 @@ private void getPartitionInfo(List tableList, tableTabletIds.add(tabletId); } - long partitionId = tabletMeta.getPartitionId(); long backendId = tabletCommitInfos.get(i).getBackendId(); if (!lockContext.getTableToPartitions().containsKey(tableId)) { @@ -1044,10 +1061,6 @@ private void getPartitionInfo(List tableList, partitionToTablets.put(partitionId, Sets.newHashSet()); } partitionToTablets.get(partitionId).add(tabletId); - Partition partition = tableMap.get(tableId).getPartition(partitionId); - if (partition == null) { - throw new MetaNotFoundException("partition " + partitionId + " does not exist"); - } lockContext.getPartitions().putIfAbsent(partitionId, partition); } if (!tableList.isEmpty() && !tabletCommitInfos.isEmpty() && lockContext.getTableToTabletList().isEmpty()) { @@ -1581,6 +1594,8 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, List mowTableList = getMowTableList(tableList, tabletCommitInfos); try { Map> backendToPartitionInfos = null; + long commitTSO = TransactionUtil.getCommitTSO(transactionId, (Database) db, + tableList.stream().map(Table::getId).collect(Collectors.toSet())); if (!mowTableList.isEmpty()) { if (!checkTransactionStateBeforeCommit(db.getId(), transactionId)) { return true; @@ -1598,7 +1613,7 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, lockContext, partitionToSubTxnIds); } commitTransactionWithSubTxns(db.getId(), tableList, transactionId, subTransactionStates, mowTableList, - backendToPartitionInfos); + backendToPartitionInfos, commitTSO); // clear signature after commit succeeds clearTxnLastSignature(db.getId(), transactionId); } catch (Exception e) { @@ -1636,7 +1651,8 @@ public boolean commitAndPublishTransaction(DatabaseIf db, long transactionId, private void commitTransactionWithSubTxns(long dbId, List
tableList, long transactionId, List subTransactionStates, List mowTableList, - Map> backendToPartitionInfos) throws UserException { + Map> backendToPartitionInfos, long commitTSO) + throws UserException { if (!mowTableList.isEmpty()) { List mowTableIds = mowTableList.stream().map(Table::getId).collect(Collectors.toList()); sendCalcDeleteBitmaptask(dbId, transactionId, backendToPartitionInfos, mowTableIds, @@ -1651,6 +1667,7 @@ private void commitTransactionWithSubTxns(long dbId, List
tableList, long .setIs2Pc(false) .setCloudUniqueId(Config.cloud_unique_id) .setIsTxnLoad(true) + .setCommitTso(commitTSO) .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit); for (OlapTable olapTable : mowTableList) { builder.addMowTableIds(olapTable.getId()); @@ -1682,9 +1699,11 @@ private List
getTablesNeedCommitLock(List
tableList) { .sorted(Comparator.comparingLong(Table::getId)) .collect(Collectors.toList()); } else { - // If disabled, only lock MOW tables + // If disabled, only lock tables that need ordered commit metadata. return tableList.stream() - .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) + .filter(table -> table instanceof OlapTable + && (((OlapTable) table).getEnableUniqueKeyMergeOnWrite() + || ((OlapTable) table).enableTso())) .sorted(Comparator.comparingLong(Table::getId)) .collect(Collectors.toList()); } @@ -1722,6 +1741,26 @@ private void beforeCommitTransaction(List
tableList, long transactionId, } } } + if (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait")) { + LOG.info("debug point: block at CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait"); + DebugPoint debugPoint = DebugPointUtil.getDebugPoint( + "CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait"); + String token = debugPoint.param("token", "invalid_token"); + while (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait")) { + DebugPoint spinWaitDebugPoint = DebugPointUtil.getDebugPoint( + "CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait"); + String passToken = spinWaitDebugPoint.param("pass_token", ""); + if (token.equals(passToken)) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + LOG.info("error ", e); + } + } + LOG.info("debug point: leave CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait"); + } if (DebugPointUtil.isEnable("CloudGlobalTransactionMgr.tryCommitLock.timeout")) { DebugPoint debugPoint = DebugPointUtil.getDebugPoint( "CloudGlobalTransactionMgr.tryCommitLock.timeout"); @@ -1817,13 +1856,21 @@ public boolean commitAndPublishTransaction(DatabaseIf db, List
tableList, @Override public void commitTransaction2PC(Database db, List
tableList, long transactionId, long timeoutMillis) throws UserException { - List mowTableList = getMowTableList(tableList, null); - if (!mowTableList.isEmpty()) { - if (!checkTransactionStateBeforeCommit(db.getId(), transactionId)) { - return; + beforeCommitTransaction(tableList, transactionId, timeoutMillis); + try { + List mowTableList = getMowTableList(tableList, null); + if (!mowTableList.isEmpty()) { + if (!checkTransactionStateBeforeCommit(db.getId(), transactionId)) { + return; + } } + long commitTSO = TransactionUtil.getCommitTSO(transactionId, db, + tableList.stream().map(Table::getId).collect(Collectors.toSet())); + commitTransactionWithoutLock(db.getId(), tableList, transactionId, null, null, true, + mowTableList, null, commitTSO); + } finally { + afterCommitTransaction(tableList, transactionId); } - commitTransactionWithoutLock(db.getId(), tableList, transactionId, null, null, true, mowTableList, null); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java index 51dd6d0dc1af27..25781f16c783a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java @@ -40,7 +40,7 @@ public class IndexInfoProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("IndexId").add("IndexName").add("SchemaVersion").add("SchemaHash") - .add("ShortKeyColumnCount").add("StorageType").add("Keys").build(); + .add("ShortKeyColumnCount").add("StorageType").add("Keys").add("IsRowBinlog").build(); private DatabaseIf db; private TableIf table; @@ -65,12 +65,12 @@ public ProcResult fetchResult() throws AnalysisException { // indices order List indices = Lists.newArrayList(); indices.add(olapTable.getBaseIndexId()); - indices.addAll(olapTable.getIndexIdListExceptBaseIndex()); + indices.addAll(olapTable.getIndexIdListWithRowBinlogExceptBaseIndex()); for (long indexId : indices) { - MaterializedIndexMeta indexMeta = olapTable.getIndexIdToMeta().get(indexId); + MaterializedIndexMeta indexMeta = olapTable.getIndexIdToMetaWithRowBinlog().get(indexId); - String type = olapTable.getIndexMetaByIndexId(indexId).getKeysType().name(); + String type = indexMeta.getKeysType().name(); StringBuilder builder = new StringBuilder(); builder.append(type).append("("); List columnNames = Lists.newArrayList(); @@ -88,10 +88,12 @@ public ProcResult fetchResult() throws AnalysisException { String.valueOf(indexMeta.getSchemaHash()), String.valueOf(indexMeta.getShortKeyColumnCount()), indexMeta.getStorageType().name(), - builder.toString())); + builder.toString(), + String.valueOf(indexMeta.isRowBinlogIndex()))); } } else { - result.addRow(Lists.newArrayList(String.valueOf(table.getId()), table.getName(), "", "", "", "", "")); + result.addRow(Lists.newArrayList(String.valueOf(table.getId()), table.getName(), "", "", "", "", "", + "false")); } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 4bf00c0be67332..049172f8b3ac30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -114,7 +114,7 @@ public class PartitionsProcDir implements ProcDirInterface { .add("Buckets").add("ReplicationNum").add("StorageMedium").add("CooldownTime").add("RemoteStoragePolicy") .add("LastConsistencyCheckTime").add("DataSize").add("IsInMemory").add("ReplicaAllocation") .add("IsMutable").add("SyncWithBaseTables").add("UnsyncTables").add("CommittedVersion") - .add("RowCount") + .add("RowCount").add("BinlogSize") .build(); private Database db; @@ -651,6 +651,13 @@ private List, TRow>> getPartitionInfosInrernal() throws An partitionInfo.add(partition.getRowCount()); trow.addToColumnValue(new TCell().setLongVal(partition.getRowCount())); + long binlogSize = partition.getBinlogDataSize(); + Pair binlogSizePair = DebugUtil.getByteUint(binlogSize); + String readableBinlogSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(binlogSizePair.first) + " " + + binlogSizePair.second; + partitionInfo.add(readableBinlogSize); + trow.addToColumnValue(new TCell().setStringVal(readableBinlogSize)); + partitionInfos.add(Pair.of(partitionInfo, trow)); } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java index b75b0d8ee953c0..fb4d5cbd5d42bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java @@ -53,8 +53,7 @@ public class ReplicasProcNode implements ProcNodeInterface { .add("IsUserDrop") .add("VisibleVersionCount").add("VersionCount").add("PathHash").add("Path") .add("MetaUrl").add("CompactionStatus").add("CooldownReplicaId") - .add("CooldownMetaId").add("QueryHits").add("BinlogSize").add("BinlogFileNum") - .add("WindowAccessCount").add("LastAccessTime"); + .add("CooldownMetaId").add("QueryHits").add("WindowAccessCount").add("LastAccessTime"); if (Config.isCloudMode()) { builder.add("PrimaryBackendId"); @@ -158,8 +157,6 @@ public ProcResult fetchResult() throws AnalysisException { String.valueOf(tablet.getCooldownReplicaId()), cooldownMetaId, String.valueOf(queryHits), - String.valueOf(replica.getBinlogSize()), - String.valueOf(replica.getBinlogFileNum()), String.valueOf(accessCount), String.valueOf(lastAccessTime) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java index 46c41dabebca7b..d102fb6aab0e6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java @@ -193,7 +193,7 @@ static class DBTabletStatistic { ReplicaAllocation replicaAlloc = olapTable.getPartitionInfo() .getReplicaAllocation(partition.getId()); for (MaterializedIndex materializedIndex : partition.getMaterializedIndices( - MaterializedIndex.IndexExtState.VISIBLE)) { + MaterializedIndex.IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { List tablets = materializedIndex.getTablets(); for (int i = 0; i < tablets.size(); ++i) { Tablet tablet = tablets.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index 8a8408a91c6cc5..676942d0b1b521 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -67,8 +67,7 @@ public class TabletsProcDir implements ProcDirInterface { .add("VisibleVersionCount").add("VersionCount").add("QueryHits").add("WindowAccessCount") .add("LastAccessTime").add("PathHash").add("Path") .add("MetaUrl").add("CompactionStatus") - .add("CooldownReplicaId").add("CooldownMetaId") - .add("BinlogSize").add("BinlogFileNum"); + .add("CooldownReplicaId").add("CooldownMetaId").add("IsRowBinlog"); if (Config.isCloudMode()) { builder.add("PrimaryBackendId"); } @@ -158,8 +157,7 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(FeConstants.null_string); // compaction status tabletInfo.add(-1); // cooldown replica id tabletInfo.add(""); // cooldown meta id - tabletInfo.add(-1L); // binlog data size - tabletInfo.add(-1L); // binlog file num + tabletInfo.add(index.isRowBinlog()); // is row binlog if (Config.isCloudMode()) { tabletInfo.add(-1L); // primary backend id } @@ -227,8 +225,7 @@ public List> fetchComparableResult(long version, long backendId } else { tabletInfo.add(replica.getCooldownMetaId().toString()); } - tabletInfo.add(replica.getBinlogSize()); - tabletInfo.add(replica.getBinlogFileNum()); + tabletInfo.add(index.isRowBinlog()); if (Config.isCloudMode()) { tabletInfo.add(((CloudReplica) replica).getPrimaryBackendId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BufferSizeUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BufferSizeUtil.java index d8194861f7425c..2ea91a4dce6fc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BufferSizeUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BufferSizeUtil.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -37,6 +38,10 @@ public static long getBufferSizeForCreateTable(CreateTableInfo createTableInfo, long partitionNum = createTableInfo.getPartitionDesc() == null ? 1 : createTableInfo.getPartitionDesc().getSinglePartitionDescs().size(); long indexNum = createTableInfo.getAddRollupOps().size() + 1; + // The row binlog companion index needs its own index/tablet/replica ids reserved. + if (BinlogConfig.fromProperties(createTableInfo.getProperties()).isEnableForStreaming()) { + indexNum++; + } long bucketNum = createTableInfo.getDistributionDesc().toDistributionInfo(createTableInfo.getColumns()) .getBucketNum(); bufferSize = bufferSize + partitionNum + indexNum; @@ -48,8 +53,6 @@ public static long getBufferSizeForCreateTable(CreateTableInfo createTableInfo, bufferSize = bufferSize + (replicaNum + 1) * indexNum * bucketNum; } } - // Reserve one extra id for binlog index id. - bufferSize += 1; return bufferSize; } @@ -58,12 +61,10 @@ public static long getBufferSizeForTruncateTable(OlapTable table, Collection entry : olapTable.getIndexIdToMeta().entrySet()) { + for (Map.Entry entry + : olapTable.getIndexIdToMetaWithRowBinlog().entrySet()) { long indexId = entry.getKey(); if (!indexIdToMeta.containsKey(indexId)) { metaChanged = true; @@ -2094,7 +2095,9 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa Partition partition = new Partition(partitionId, partitionName, baseIndex, distributionInfo); // add to index map - Map indexMap = new HashMap<>(); + // Use LinkedHashMap so the base index is processed before the row binlog companion index: + // the companion tablets are derived from base tablets, which must be created first. + Map indexMap = Maps.newLinkedHashMap(); indexMap.put(tbl.getBaseIndexId(), baseIndex); // create rollup index if has @@ -2103,7 +2106,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa continue; } - MaterializedIndex rollup = new MaterializedIndex(indexId, IndexState.NORMAL); + MaterializedIndex rollup = new MaterializedIndex(indexId, + indexIdToMeta.get(indexId).isRowBinlogIndex() ? IndexState.ROW_BINLOG : IndexState.NORMAL); indexMap.put(indexId, rollup); } @@ -2121,14 +2125,20 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa long indexId = entry.getKey(); MaterializedIndex index = entry.getValue(); MaterializedIndexMeta indexMeta = indexIdToMeta.get(indexId); + boolean isRowBinlogIndex = indexMeta.isRowBinlogIndex(); // create tablets int schemaHash = indexMeta.getSchemaHash(); TabletMeta tabletMeta = new TabletMeta(dbId, tbl.getId(), partitionId, indexId, schemaHash, dataProperty.getStorageMedium()); - realStorageMedium = createTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, - tabletMeta, tabletIdSet, idGeneratorBuffer, dataProperty.isStorageMediumSpecified()); - if (realStorageMedium != null && !realStorageMedium.equals(dataProperty.getStorageMedium())) { + if (isRowBinlogIndex) { + createRowBinlogTablets(index, baseIndex, version, tabletMeta, tabletIdSet, idGeneratorBuffer); + } else { + realStorageMedium = createTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, + tabletMeta, tabletIdSet, idGeneratorBuffer, dataProperty.isStorageMediumSpecified()); + } + if (!isRowBinlogIndex && realStorageMedium != null + && !realStorageMedium.equals(dataProperty.getStorageMedium())) { dataProperty.setStorageMedium(realStorageMedium); LOG.info("real medium not eq default " + "tableName={} tableId={} partitionName={} partitionId={} readMedium {}", @@ -2148,6 +2158,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa clusterKeyUids = OlapTable.getClusterKeyUids(schema); } KeysType keysType = indexMeta.getKeysType(); + boolean enableUniqueKeyMergeOnWrite = !isRowBinlogIndex && tbl.getEnableUniqueKeyMergeOnWrite(); List indexes = indexId == tbl.getBaseIndexId() ? tbl.getCopiedIndexes() : null; int totalTaskNum = index.getTablets().size() * totalReplicaNum; MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalTaskNum); @@ -2155,20 +2166,27 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa List rowStoreColumns = tbl.getTableProperty().getCopiedRowStoreColumns(); for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); + Tablet baseTablet = null; + TStorageMedium taskMedium = realStorageMedium; + if (isRowBinlogIndex) { + // The binlog tablet records its base tablet id via alignedTabletId. + baseTablet = baseIndex.getTablet(tablet.getAlignedTabletId()); + Preconditions.checkNotNull(baseTablet, + "row binlog tablet %s's base tablet %s can not be found in meta", + tabletId, tablet.getAlignedTabletId()); + // The companion tablet shares the partition data medium with its base tablet (same disk). + taskMedium = dataProperty.getStorageMedium(); + } for (Replica replica : tablet.getReplicas()) { long backendId = replica.getBackendIdWithoutException(); long replicaId = replica.getId(); countDownLatch.addMark(backendId, tabletId); - MaterializedIndexMeta rowBinlogIndexMeta = null; - if (tbl.needRowBinlog() && indexId == tbl.getBaseIndexId()) { - rowBinlogIndexMeta = tbl.getRowBinlogMeta(); - } CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tbl.getId(), partitionId, indexId, tabletId, replicaId, shortKeyColumnCount, schemaHash, version, keysType, storageType, - realStorageMedium, schema, bfColumns, tbl.getBfFpp(), countDownLatch, + taskMedium, schema, bfColumns, tbl.getBfFpp(), countDownLatch, indexes, tbl.isInMemory(), tabletType, tbl.getDataSortInfo(), tbl.getCompressionType(), - tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), + enableUniqueKeyMergeOnWrite, storagePolicy, tbl.disableAutoCompaction(), tbl.skipWriteIndexOnLoad(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), @@ -2182,8 +2200,12 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.storagePageSize(), tbl.getTDEAlgorithm(), tbl.storageDictPageSize(), tbl.getColumnSeqMapping(), - tbl.getVerticalCompactionNumColumnsPerGroup(), - rowBinlogIndexMeta); + tbl.getVerticalCompactionNumColumnsPerGroup()); + if (isRowBinlogIndex) { + // BE locates the companion binlog tablet via base_tablet_id and writes it to the same disk. + task.setIsRowBinlogTablet(true); + task.setBaseTablet(baseTablet.getId(), tbl.getBaseIndexMeta().getSchemaHash()); + } task.setStorageFormat(tbl.getStorageFormat()); task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); @@ -2830,6 +2852,12 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th if (binlogConfigMap != null) { BinlogConfig binlogConfig = new BinlogConfig(); binlogConfig.mergeFromProperties(binlogConfigMap); + if (binlogConfig.isEnableForCCR()) { + if (Config.isCloudMode()) { + throw new AnalysisException("Binlog is not supported in cloud mode"); + } + } + if (binlogConfig.isEnableForStreaming()) { if (!(keysType == KeysType.DUP_KEYS || (keysType == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite))) { @@ -2837,9 +2865,6 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th + "if you want to use mor or aggregate table model, " + "please use binlog with snapshot"); } - if (Config.isCloudMode()) { - throw new AnalysisException("Binlog is not supported in the cloud mode yet"); - } if (keysType == KeysType.DUP_KEYS && binlogConfig.getNeedHistoricalValue()) { throw new AnalysisException("Duplicate table model don't support record historical value"); } @@ -3081,7 +3106,7 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th long partitionId = partitionNameToId.get(partitionName); // check replica quota if this operation done - long indexNum = olapTable.getIndexIdToMeta().size(); + long indexNum = olapTable.getIndexNumberWithRowBinlog(); long bucketNum = partitionDistributionInfo.getBucketNum(); long replicaNum = partitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum(); long totalReplicaNum = indexNum * bucketNum * replicaNum; @@ -3091,10 +3116,11 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th + " increasing " + totalReplicaNum + " of replica exceeds quota[" + db.getReplicaQuota() + "]"); } - beforeCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); + beforeCreatePartitions(db.getId(), olapTable.getId(), null, + olapTable.getIndexIdListWithRowBinlog(), true); Partition partition = createPartitionWithIndices(db.getId(), olapTable, partitionId, partitionName, - olapTable.getIndexIdToMeta(), partitionDistributionInfo, + olapTable.getIndexIdToMetaWithRowBinlog(), partitionDistributionInfo, partitionInfo.getDataProperty(partitionId), partitionInfo.getReplicaAllocation(partitionId), versionInfo, bfColumns, tabletIdSet, isInMemory, tabletType, @@ -3104,7 +3130,8 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified()); olapTable.addPartition(partition); afterCreatePartitions(db.getId(), olapTable.getId(), olapTable.getPartitionIds(), - olapTable.getIndexIdList(), true /* isCreateTable */, true /* isBatchCommit */, olapTable); + olapTable.getIndexIdListWithRowBinlog(), + true /* isCreateTable */, true /* isBatchCommit */, olapTable); } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { try { @@ -3151,7 +3178,7 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th // check replica quota if this operation done long totalReplicaNum = 0; for (Map.Entry entry : partitionNameToId.entrySet()) { - long indexNum = olapTable.getIndexIdToMeta().size(); + long indexNum = olapTable.getIndexNumberWithRowBinlog(); long bucketNum = defaultDistributionInfo.getBucketNum(); long replicaNum = partitionInfo.getReplicaAllocation(entry.getValue()).getTotalReplicaNum(); totalReplicaNum += indexNum * bucketNum * replicaNum; @@ -3162,7 +3189,8 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th + totalReplicaNum + " of replica exceeds quota[" + db.getReplicaQuota() + "]"); } - beforeCreatePartitions(db.getId(), olapTable.getId(), null, olapTable.getIndexIdList(), true); + beforeCreatePartitions(db.getId(), olapTable.getId(), null, + olapTable.getIndexIdListWithRowBinlog(), true); // this is a 2-level partitioned tables for (Map.Entry entry : partitionNameToId.entrySet()) { @@ -3185,7 +3213,7 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy); Partition partition = createPartitionWithIndices(db.getId(), olapTable, entry.getValue(), - entry.getKey(), olapTable.getIndexIdToMeta(), partitionDistributionInfo, + entry.getKey(), olapTable.getIndexIdToMetaWithRowBinlog(), partitionDistributionInfo, dataProperty, partitionInfo.getReplicaAllocation(entry.getValue()), versionInfo, bfColumns, tabletIdSet, isInMemory, partitionInfo.getTabletType(entry.getValue()), @@ -3197,7 +3225,8 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th .setStoragePolicy(partionStoragePolicy); } afterCreatePartitions(db.getId(), olapTable.getId(), olapTable.getPartitionIds(), - olapTable.getIndexIdList(), true /* isCreateTable */, true /* isBatchCommit */, olapTable); + olapTable.getIndexIdListWithRowBinlog(), + true /* isCreateTable */, true /* isBatchCommit */, olapTable); } else { throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name()); } @@ -3452,6 +3481,32 @@ public TStorageMedium createTablets(MaterializedIndex index, ReplicaState replic return storageMedium; } + // Create the companion row-binlog tablets aligned to base tablets. Each companion reuses its + // base replica backends so BE can place it on the same disk (located via base_tablet_id in the + // CreateReplicaTask). The base <-> binlog tablet ids are cross-linked via alignedTabletId. + private void createRowBinlogTablets(MaterializedIndex rowBinlogIndex, MaterializedIndex baseIndex, + long version, TabletMeta tabletMeta, Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer) { + TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex(); + List baseTablets = baseIndex.getTablets(); + List rowBinlogTablets = new ArrayList<>(baseTablets.size()); + for (Tablet baseTablet : baseTablets) { + Tablet rowBinlogTablet = EnvFactory.getInstance().createTablet(idGeneratorBuffer.getNextId()); + baseTablet.setAlignedTabletId(rowBinlogTablet.getId()); + rowBinlogTablet.setAlignedTabletId(baseTablet.getId()); + invertedIndex.addTablet(rowBinlogTablet.getId(), tabletMeta); + rowBinlogTablets.add(rowBinlogTablet); + tabletIdSet.add(rowBinlogTablet.getId()); + + for (Replica baseReplica : baseTablet.getReplicas()) { + long replicaId = idGeneratorBuffer.getNextId(); + Replica replica = new LocalReplica(replicaId, baseReplica.getBackendIdWithoutException(), + ReplicaState.NORMAL, version, tabletMeta.getOldSchemaHash()); + rowBinlogTablet.addReplica(replica); + } + } + rowBinlogIndex.appendTablets(rowBinlogTablets); + } + /* * generate and check columns' order and key's existence, */ @@ -3607,7 +3662,7 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa newPartitionIds.add(newPartitionId); } - List indexIds = copiedTbl.getIndexIdToMeta().keySet().stream().collect(Collectors.toList()); + List indexIds = copiedTbl.getIndexIdListWithRowBinlog(); beforeCreatePartitions(db.getId(), copiedTbl.getId(), newPartitionIds, indexIds, true); for (Map.Entry entry : origPartitions.entrySet()) { @@ -3620,7 +3675,7 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa long newPartitionId = oldToNewPartitionId.get(oldPartitionId); Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, newPartitionId, entry.getKey(), - copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), + copiedTbl.getIndexIdToMetaWithRowBinlog(), partitionsDistributionInfo.get(oldPartitionId), copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId), copiedTbl.getPartitionInfo().getReplicaAllocation(oldPartitionId), null /* version info */, copiedTbl.getCopiedBfColumns(), tabletIdSet, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java index e4971a96b7bbc3..a051fc6accc0f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java @@ -279,6 +279,7 @@ public void generatePlan(OlapTable table) throws UserException { result.setTableName(table.getName()); result.query_options.setFeProcessUuid(ExecuteEnv.getInstance().getProcessUUID()); result.setIsMowTable(table.getEnableUniqueKeyMergeOnWrite()); + result.setEnableTso(table.enableTso()); fragmentParams.add(result); if (StringUtils.isEmpty(streamLoadTask.getGroupCommit())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index dde42c36ca7b15..acbe090543ff48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -27,7 +27,6 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.LocalReplica; import org.apache.doris.catalog.MaterializedIndex; -import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -991,7 +990,7 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta if (index == null) { continue; } - if (index.getState() == IndexState.SHADOW) { + if (index.getState().isShadow()) { // This index is under schema change or rollup, tablet may not be created on BE. // ignore it. continue; @@ -1060,10 +1059,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta ? olapTable.getCopiedIndexes() : null; List rowStoreColumns = olapTable.getTableProperty().getCopiedRowStoreColumns(); - MaterializedIndexMeta rowBinlogIndexMeta = null; - if (olapTable.needRowBinlog() && indexId == olapTable.getBaseIndexId()) { - rowBinlogIndexMeta = olapTable.getRowBinlogMeta(); - } CreateReplicaTask createReplicaTask = new CreateReplicaTask(backendId, dbId, tableId, partitionId, indexId, tabletId, replica.getId(), indexMeta.getShortKeyColumnCount(), @@ -1093,11 +1088,21 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta olapTable.storagePageSize(), olapTable.getTDEAlgorithm(), olapTable.storageDictPageSize(), olapTable.getColumnSeqMapping(), - olapTable.getVerticalCompactionNumColumnsPerGroup(), - rowBinlogIndexMeta); + olapTable.getVerticalCompactionNumColumnsPerGroup()); createReplicaTask.setIsRecoverTask(true); createReplicaTask.setInvertedIndexFileStorageFormat(olapTable .getInvertedIndexFileStorageFormat()); + if (indexMeta.isRowBinlogIndex()) { + Tablet baseTablet = partition.getBaseIndex() + .getTablet(tablet.getAlignedTabletId()); + Preconditions.checkNotNull(baseTablet, + "row binlog tablet %s's base tablet %s can not be found " + + "in partition %s", + tablet.getId(), tablet.getAlignedTabletId(), partition.getId()); + createReplicaTask.setIsRowBinlogTablet(true); + createReplicaTask.setBaseTablet(baseTablet.getId(), + olapTable.getBaseIndexMeta().getSchemaHash()); + } if (indexId == olapTable.getBaseIndexId() || olapTable.isShadowIndex(indexId)) { List clusterKeyUids = OlapTable.getClusterKeyUids( indexMeta.getSchema()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 6bc4a4974e91ec..596c7d04fab465 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -338,6 +338,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde params.setQueryGlobals(queryGlobals); params.setTableName(destTable.getName()); params.setIsMowTable(destTable.getEnableUniqueKeyMergeOnWrite()); + params.setEnableTso(destTable.enableTso()); return params; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java index cba3ef7a726f91..0ca208575f10e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCompactTableCommand.java @@ -47,8 +47,7 @@ public class AdminCompactTableCommand extends Command implements ForwardWithSync public enum CompactionType { CUMULATIVE, BASE, - FULL, - BINLOG + FULL } private CompactionType typeFilter; @@ -89,12 +88,12 @@ private void validate(ConnectContext ctx) throws UserException { // analyze where clause if not null if (where == null) { throw new AnalysisException("Compaction type must be specified in" - + " Where clause like: type = 'BASE/CUMULATIVE/FULL/BINLOG'"); + + " Where clause like: type = 'BASE/CUMULATIVE/FULL'"); } if (!analyzeWhere()) { throw new AnalysisException( - "Where clause should looks like: type = 'BASE/CUMULATIVE/FULL/BINLOG'"); + "Where clause should looks like: type = 'BASE/CUMULATIVE/FULL'"); } } @@ -107,8 +106,7 @@ private boolean analyzeWhere() { return typeFilter == CompactionType.CUMULATIVE || typeFilter == CompactionType.BASE - || typeFilter == CompactionType.FULL - || typeFilter == CompactionType.BINLOG; + || typeFilter == CompactionType.FULL; } @Override @@ -124,8 +122,6 @@ private String getCompactionType() { return "base"; case FULL: return "full"; - case BINLOG: - return "binlog"; default: throw new IllegalStateException("unexpected compaction type: " + typeFilter); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataCommand.java index e570c9e582ca88..9b1e6799859d6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataCommand.java @@ -303,10 +303,9 @@ public int compare(Table t1, Table t2) { private void collectTableStats(OlapTable table) { // sort by index name - Map indexNames = table.getIndexNameToId(); Map sortedIndexNames = new TreeMap(); - for (Map.Entry entry : indexNames.entrySet()) { - sortedIndexNames.put(entry.getKey(), entry.getValue()); + for (Long indexId : table.getIndexIdToMetaWithRowBinlog().keySet()) { + sortedIndexNames.put(table.getIndexNameById(indexId), indexId); } for (Long indexId : sortedIndexNames.values()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommand.java index a311ad5404ff24..5c6bbbbfdbf8dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommand.java @@ -82,6 +82,7 @@ public ShowResultSetMetaData getMetaData() { builder.addColumn(new Column("QueryHits", ScalarType.createVarchar(30))); builder.addColumn(new Column("WindowAccessCount", ScalarType.createVarchar(30))); builder.addColumn(new Column("LastAccessTime", ScalarType.createVarchar(30))); + builder.addColumn(new Column("IsRowBinlog", ScalarType.createVarchar(30))); builder.addColumn(new Column("DetailCmd", ScalarType.createVarchar(30))); return builder.build(); } @@ -111,6 +112,7 @@ private ShowResultSet handleShowTabletId() { Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; String indexName = FeConstants.null_string; Boolean isSync = true; + Boolean isRowBinlog = false; long queryHits = 0L; int tabletIdx = -1; @@ -162,6 +164,7 @@ private ShowResultSet handleShowTabletId() { break; } indexName = olapTable.getIndexNameById(indexId); + isRowBinlog = index.isRowBinlog(); Tablet tablet = index.getTablet(tabletId); if (tablet == null) { @@ -197,7 +200,7 @@ private ShowResultSet handleShowTabletId() { partitionId.toString(), indexId.toString(), isSync.toString(), String.valueOf(tabletIdx), String.valueOf(queryHits), String.valueOf(accessCount), String.valueOf(lastAccessTime), - detailCmd)); + isRowBinlog.toString(), detailCmd)); return new ShowResultSet(getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 0e4f45c94d16bf..f34d28820213e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -216,6 +216,11 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou LOG.debug("Single replica load not supported by merge-on-write table: {}", dstTable.getName()); } } + if (singleReplicaLoad && dstTable.needRowBinlog()) { + // Single replica load not supported by table with row binlog + singleReplicaLoad = false; + LOG.warn("Single replica load not supported by table with row binlog: {}", dstTable.getName()); + } } // init for nereids insert into @@ -226,10 +231,10 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou isStrictMode, txnExpirationS); for (Long partitionId : partitionIds) { Partition partition = dstTable.getPartition(partitionId); - if (dstTable.getIndexNumber() != partition.getMaterializedIndices(IndexExtState.ALL).size()) { + if (dstTable.getIndexNumberWithRowBinlog() != partition.getMaterializedIndices(IndexExtState.ALL).size()) { throw new UserException( "table's index number not equal with partition's index number. table's index number=" - + dstTable.getIndexIdToMeta().size() + ", partition's index number=" + + dstTable.getIndexNumberWithRowBinlog() + ", partition's index number=" + partition.getMaterializedIndices(IndexExtState.ALL).size()); } } @@ -284,10 +289,10 @@ public void init(TUniqueId loadId, long txnId, long dbId, long loadChannelTimeou isStrictMode, txnExpirationS); for (Long partitionId : partitionIds) { Partition partition = dstTable.getPartition(partitionId); - if (dstTable.getIndexNumber() != partition.getMaterializedIndices(IndexExtState.ALL).size()) { + if (dstTable.getIndexNumberWithRowBinlog() != partition.getMaterializedIndices(IndexExtState.ALL).size()) { throw new UserException( "table's index number not equal with partition's index number. table's index number=" - + dstTable.getIndexIdToMeta().size() + ", partition's index number=" + + dstTable.getIndexNumberWithRowBinlog() + ", partition's index number=" + partition.getMaterializedIndices(IndexExtState.ALL).size()); } } @@ -472,7 +477,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An TOlapTableIndexSchema rowBinlogIndexSchema = new TOlapTableIndexSchema( rowBinlogMeta.getIndexId(), binlogColumns, rowBinlogMeta.getSchemaHash()); rowBinlogIndexSchema.setColumnsDesc(binlogColumnsDesc); - schemaParam.setRowBinlogIndexSchema(rowBinlogIndexSchema); + schemaParam.addToRowBinlogIndexSchemas(rowBinlogIndexSchema); } setPartialUpdateInfoForParam(schemaParam, table, uniqueKeyUpdateMode); @@ -1022,7 +1027,8 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) // set partition keys setPartitionKeys(tPartition, partitionInfo.getItem(partition.getId()), partColNum); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.ALL_EXCEPT_ROW_BINLOG)) { tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); tPartition.setNumBuckets(index.getTablets().size()); @@ -1086,7 +1092,8 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) tPartition.setId(partition.getId()); tPartition.setIsMutable(table.getPartitionInfo().getIsMutable(partition.getId())); // No lowerBound and upperBound for this range - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { + for (MaterializedIndex index + : partition.getMaterializedIndices(IndexExtState.ALL_EXCEPT_ROW_BINLOG)) { tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); tPartition.setNumBuckets(index.getTablets().size()); @@ -1197,6 +1204,17 @@ public List createDummyLocation(OlapTable table) throws return Arrays.asList(locationParam, slaveLocationParam); } + // In non-cloud mode the binlog tablet must be on the same disk as its base tablet (cloud mode + // does not need this), so keep only the (backend, pathHash) entries shared by both tablets. + private Multimap getBinlogColocatedReplicaBackendPathMap(Tablet baseTablet, Tablet rowBinlogTablet) + throws UserException { + Multimap baseBePathsMap = baseTablet.getNormalReplicaBackendPathMap(); + Multimap binlogBePathsMap = rowBinlogTablet.getNormalReplicaBackendPathMap(); + binlogBePathsMap.entries().removeIf( + entry -> !baseBePathsMap.containsEntry(entry.getKey(), entry.getValue())); + return binlogBePathsMap; + } + private List createLocation(long dbId, OlapTable table) throws UserException { if (table.getPartitionInfo().enableAutomaticPartition() && partitionIds.isEmpty()) { return createDummyLocation(table); @@ -1225,6 +1243,12 @@ private List createLocation(long dbId, OlapTable table) } bePathsMap = ((CloudTablet) tablet) .getNormalReplicaBackendPathMapByClusterId(cachedClusterId); + } else if (index.isRowBinlog()) { + Tablet baseTablet = partition.getBaseIndex().getTablet(tablet.getAlignedTabletId()); + Preconditions.checkNotNull(baseTablet, + "row binlog tablet %s's base tablet %s can not be found in partition %s", + tablet.getId(), tablet.getAlignedTabletId(), partition.getId()); + bePathsMap = getBinlogColocatedReplicaBackendPathMap(baseTablet, tablet); } else { bePathsMap = tablet.getNormalReplicaBackendPathMap(); } @@ -1240,6 +1264,15 @@ private List createLocation(long dbId, OlapTable table) // and each cluster has only one replica, no need to detail the replicas in cloud mode. errMsgBuilder.append(", detail: ") .append(tablet.getDetailsStatusForQuery(partition.getVisibleVersion())); + if (index.isRowBinlog()) { + // replica num is counted after intersecting with the base tablet for same-disk + // co-location, so also surface the base tablet status for diagnosis. + Tablet baseTablet = partition.getBaseIndex() + .getTablet(tablet.getAlignedTabletId()); + errMsgBuilder.append(", base tablet ").append(tablet.getAlignedTabletId()) + .append(" detail: ") + .append(baseTablet.getDetailsStatusForQuery(partition.getVisibleVersion())); + } } long now = System.currentTimeMillis(); long lastLoadFailedTime = tablet.getLastLoadFailedTime(); @@ -1269,13 +1302,24 @@ private List createLocation(long dbId, OlapTable table) Long masterNode = nodes[random.nextInt(nodes.length)]; Multimap slaveBePathsMap = bePathsMap; slaveBePathsMap.removeAll(masterNode); - locationParam.addToTablets(new TTabletLocation(tablet.getId(), - Lists.newArrayList(Sets.newHashSet(masterNode)))); - slaveLocationParam.addToTablets(new TTabletLocation(tablet.getId(), - Lists.newArrayList(slaveBePathsMap.keySet()))); + TTabletLocation location = new TTabletLocation(tablet.getId(), + Lists.newArrayList(Sets.newHashSet(masterNode))); + TTabletLocation slaveLocation = new TTabletLocation(tablet.getId(), + Lists.newArrayList(slaveBePathsMap.keySet())); + if (index.isRowBinlog()) { + location.setBaseTabletId(tablet.getAlignedTabletId()); + slaveLocation.setBaseTabletId(tablet.getAlignedTabletId()); + } + locationParam.addToTablets(location); + slaveLocationParam.addToTablets(slaveLocation); } else { - locationParam.addToTablets(new TTabletLocation(tablet.getId(), - Lists.newArrayList(bePathsMap.keySet()))); + TTabletLocation location = new TTabletLocation(tablet.getId(), + Lists.newArrayList(bePathsMap.keySet())); + if (index.isRowBinlog()) { + // BE pairs the binlog tablet with its base tablet via base_tablet_id. + location.setBaseTabletId(tablet.getAlignedTabletId()); + } + locationParam.addToTablets(location); } allBePathsMap.putAll(bePathsMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/InsertStreamTxnExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/InsertStreamTxnExecutor.java index df4fee5fbfb403..30176665f287c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/InsertStreamTxnExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/InsertStreamTxnExecutor.java @@ -80,6 +80,7 @@ public void beginTransaction(TStreamLoadPutRequest request) throws UserException TPipelineFragmentParams tRequest = planner.plan(streamLoadTask.getId()); tRequest.setTxnConf(txnConf).setImportLabel(txnEntry.getLabel()); tRequest.setIsMowTable(isMowTable); + tRequest.setEnableTso(table.enableTso()); for (Map.Entry> entry : tRequest.local_params.get(0).per_node_scan_ranges .entrySet()) { for (TScanRangeParams scanRangeParams : entry.getValue()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index b737ed2bc9496e..f7e4a25e56ecef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3228,6 +3228,7 @@ private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResu result.getPipelineParams().setImportLabel(httpStreamParams.getLabel()); result.getPipelineParams() .setIsMowTable(((OlapTable) httpStreamParams.getTable()).getEnableUniqueKeyMergeOnWrite()); + result.getPipelineParams().setEnableTso(((OlapTable) httpStreamParams.getTable()).enableTso()); result.setDbId(httpStreamParams.getDb().getId()); result.setTableId(httpStreamParams.getTable().getId()); result.setBaseSchemaVersion(((OlapTable) httpStreamParams.getTable()).getBaseSchemaVersion()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java index 52f207261ba821..157355e468109c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableBinlogFunction.java @@ -28,7 +28,6 @@ import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanNodeId; @@ -71,11 +70,6 @@ public class TableBinlogFunction extends TableValuedFunctionIf { private final RowBinlogTableWrapper rowBinlogTableWrapper; public TableBinlogFunction(Map params) throws AnalysisException { - // Cloud mode uses a single-version snapshot for scan range versioning and is not supported here. - if (Config.isCloudMode()) { - throw new AnalysisException("binlog table valued function is not supported in cloud mode"); - } - Map validParams = Maps.newHashMap(); for (Map.Entry e : params.entrySet()) { String key = StringUtils.lowerCase(e.getKey()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java index db10cf83de49ad..d1c1ed177345fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java @@ -17,7 +17,6 @@ package org.apache.doris.task; -import org.apache.doris.catalog.Tablet.CopyType; import org.apache.doris.thrift.TBackend; import org.apache.doris.thrift.TCloneReq; import org.apache.doris.thrift.TStorageMedium; @@ -45,8 +44,6 @@ public class CloneTask extends AgentTask { private int taskVersion = VERSION_1; - private int copyType = CopyType.DEFAULT; - public CloneTask(TBackend destBackend, long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, long replicaId, int schemaHash, List srcBackends, TStorageMedium storageMedium, long visibleVersion, int timeoutS) { @@ -82,11 +79,6 @@ public int getTaskVersion() { return taskVersion; } - public void setCopyType(int copyType) { - CopyType.validate(copyType); - this.copyType = copyType; - } - public TCloneReq toThrift() { TCloneReq request = new TCloneReq(tabletId, schemaHash, srcBackends); request.setReplicaId(replicaId); @@ -100,7 +92,6 @@ public TCloneReq toThrift() { } request.setTimeoutS(timeoutS); request.setTableId(tableId); - request.setCopyType(copyType); return request; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 32fe0946d8d782..32fe041e3858be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -25,10 +25,10 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.IndexToThriftConvertor; import org.apache.doris.catalog.KeysType; -import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.Status; import org.apache.doris.common.util.ColumnsUtil; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.policy.Policy; import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.thrift.TColumn; @@ -131,8 +131,8 @@ public class CreateReplicaTask extends AgentTask { private boolean storeRowColumn; private BinlogConfig binlogConfig; - // update binlog schema only when create base index - private MaterializedIndexMeta rowBinlogMeta; + // whether this tablet is an independent row binlog tablet + private boolean isRowBinlogTablet = false; private List clusterKeyUids; private Map objectPool; @@ -170,8 +170,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition boolean variantEnableFlattenNested, long storagePageSize, TEncryptionAlgorithm tdeAlgorithm, long storageDictPageSize, Map> columnSeqMapping, - int verticalCompactionNumColumnsPerGroup, - MaterializedIndexMeta rowBinlogMeta) { + int verticalCompactionNumColumnsPerGroup) { super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); this.replicaId = replicaId; @@ -223,7 +222,6 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition this.storageDictPageSize = storageDictPageSize; this.tdeAlgorithm = tdeAlgorithm; this.columnSeqMapping = columnSeqMapping; - this.rowBinlogMeta = rowBinlogMeta; } public void setIsRecoverTask(boolean isRecoverTask) { @@ -285,6 +283,10 @@ public void setBaseTablet(long baseTabletId, int baseSchemaHash) { this.baseSchemaHash = baseSchemaHash; } + public void setIsRowBinlogTablet(boolean isRowBinlogTablet) { + this.isRowBinlogTablet = isRowBinlogTablet; + } + public void setStorageFormat(TStorageFormat storageFormat) { this.storageFormat = storageFormat; } @@ -448,7 +450,8 @@ public TCreateTabletReq toThrift() { createTabletReq.setTabletType(tabletType); createTabletReq.setCompressionType(compressionType); createTabletReq.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); - createTabletReq.setCompactionPolicy(compactionPolicy); + createTabletReq.setCompactionPolicy(isRowBinlogTablet + ? PropertyAnalyzer.BINLOG_COMPACTION_POLICY : compactionPolicy); createTabletReq.setTimeSeriesCompactionGoalSizeMbytes(timeSeriesCompactionGoalSizeMbytes); createTabletReq.setTimeSeriesCompactionFileCountThreshold(timeSeriesCompactionFileCountThreshold); createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds); @@ -461,46 +464,8 @@ public TCreateTabletReq toThrift() { createTabletReq.setBinlogConfig(binlogConfig.toThrift()); } - if (binlogConfig != null && binlogConfig.isEnableForStreaming() && rowBinlogMeta != null) { - TTabletSchema tRowBinlogSchema = new TTabletSchema(); - tRowBinlogSchema.setShortKeyColumnCount(rowBinlogMeta.getShortKeyColumnCount()); - tRowBinlogSchema.setSchemaHash(rowBinlogMeta.getSchemaHash()); - tRowBinlogSchema.setKeysType(rowBinlogMeta.getKeysType().toThrift()); - tRowBinlogSchema.setStorageType(TStorageType.COLUMN); - int binlogTsoIdx = -1; - int binlogLsnIdx = -1; - int binlogOpIdx = -1; - - List tRowBinlogColumns = null; - List rowBinlogColumns = rowBinlogMeta.getSchema(true); - Object tRowBinlogCols = objectPool.get(rowBinlogColumns); - if (tRowBinlogCols != null) { - tRowBinlogColumns = (List) tRowBinlogCols; - } else { - tRowBinlogColumns = new ArrayList<>(); - for (int i = 0; i < rowBinlogColumns.size(); i++) { - Column column = rowBinlogColumns.get(i); - TColumn tColumn = ColumnToThrift.toThrift(column); - tColumn.setVisible(column.isVisible()); - tRowBinlogColumns.add(tColumn); - } - objectPool.put(rowBinlogColumns, tRowBinlogColumns); - } - for (int i = 0; i < rowBinlogColumns.size(); i++) { - Column column = rowBinlogColumns.get(i); - if (column.getName().equals(Column.BINLOG_TSO_COL)) { - binlogTsoIdx = i; - } else if (column.getName().equals(Column.BINLOG_LSN_COL)) { - binlogLsnIdx = i; - } else if (column.getName().equals(Column.BINLOG_OPERATION_COL)) { - binlogOpIdx = i; - } - } - tRowBinlogSchema.setColumns(tRowBinlogColumns); - tRowBinlogSchema.setBinlogTsoIdx(binlogTsoIdx); - tRowBinlogSchema.setBinlogLsnIdx(binlogLsnIdx); - tRowBinlogSchema.setBinlogOpIdx(binlogOpIdx); - createTabletReq.setRowBinlogSchema(tRowBinlogSchema); + if (isRowBinlogTablet) { + createTabletReq.setIsRowBinlogTablet(true); } return createTabletReq; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 7f31af751823d2..495e3f61f09cdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -1601,7 +1601,8 @@ protected void unprotectedCommitTransaction(TransactionState transactionState, S // update transaction state version long commitTime = System.currentTimeMillis(); transactionState.setCommitTime(commitTime); - long commitTSO = getCommitTSO(transactionState, db, tableToPartition.keySet()); + long commitTSO = TransactionUtil.getCommitTSO(transactionState.getTransactionId(), db, + tableToPartition.keySet()); transactionState.setCommitTSO(commitTSO); if (MetricRepo.isInit) { @@ -1651,7 +1652,7 @@ protected void unprotectedCommitTransaction(TransactionState transactionState, S long tableId = subTransactionState.getTable().getId(); tableIds.add(tableId); } - long commitTSO = getCommitTSO(transactionState, db, tableIds); + long commitTSO = TransactionUtil.getCommitTSO(transactionState.getTransactionId(), db, tableIds); transactionState.setCommitTSO(commitTSO); if (MetricRepo.isInit) { @@ -1727,7 +1728,8 @@ protected void unprotectedCommitTransaction2PC(TransactionState transactionState } // update transaction state version transactionState.setCommitTime(System.currentTimeMillis()); - long commitTSO = getCommitTSO(transactionState, db, transactionState.getIdToTableCommitInfos().keySet()); + long commitTSO = TransactionUtil.getCommitTSO(transactionState.getTransactionId(), db, + transactionState.getIdToTableCommitInfos().keySet()); transactionState.setCommitTSO(commitTSO); transactionState.setTransactionStatus(TransactionStatus.COMMITTED); @@ -3112,45 +3114,6 @@ private void cleanSubTransactions(long transactionId) { } } - private long getCommitTSO(TransactionState transactionState, Database db, Set tableIds) - throws TransactionCommitFailedException { - long tso = -1L; - if (!Config.enable_feature_binlog) { - return tso; - } - if (tableIds == null || tableIds.isEmpty()) { - return tso; - } - boolean anyEnableTso = false; - for (long tableId : tableIds) { - Table table = db.getTableNullable(tableId); - if (table instanceof OlapTable && ((OlapTable) table).enableTso()) { - anyEnableTso = true; - break; - } - } - if (!anyEnableTso) { - return tso; - } - try { - Env env = Env.getCurrentEnv(); - if (env == null || env.getTSOService() == null) { - throw new TransactionCommitFailedException("failed to get TSO for txn " - + transactionState.getTransactionId() + ": TSO service is unavailable"); - } - long fetched = env.getTSOService().getTSO(); - if (fetched <= 0) { - throw new TransactionCommitFailedException("failed to get TSO for txn " - + transactionState.getTransactionId() + ", fetched=" + fetched); - } - return fetched; - } catch (RuntimeException e) { - LOG.warn("failed to get TSO for txn {}, abort commit", transactionState.getTransactionId(), e); - throw new TransactionCommitFailedException("failed to get TSO for txn " - + transactionState.getTransactionId(), e); - } - } - private List buildLockTableListNoException(List tableList, TransactionState transactionState) { if (transactionState == null || CollectionUtils.isEmpty(transactionState.getStreamUpdateInfos())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java index af5b55338a572f..ec9afc0118bea8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -708,7 +708,8 @@ public synchronized void addTableIndexes(OlapTable table) { Set indexIds = loadedTblIndexes.computeIfAbsent(table.getId(), k -> Sets.newHashSet()); // always equal the index ids indexIds.clear(); - indexIds.addAll(table.getIndexIdToMeta().keySet()); + // include the row binlog companion index so it joins commit/publish quorum + indexIds.addAll(table.getIndexIdToMetaWithRowBinlog().keySet()); } public Map> getLoadedTblIndexes() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java index 0a5fa0de5c8571..aea3478bf51f7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionUtil.java @@ -19,8 +19,10 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.TimeUtils; @@ -74,4 +76,41 @@ public static void checkAuth(long dbId, TransactionState txnState) throws Analys } } } + + public static long getCommitTSO(long transactionId, Database db, Set tableIds) + throws TransactionCommitFailedException { + long tso = -1L; + if (!Config.enable_feature_binlog) { + return tso; + } + if (tableIds == null || tableIds.isEmpty()) { + return tso; + } + boolean anyEnableTso = false; + for (long tableId : tableIds) { + Table table = db.getTableNullable(tableId); + if (table instanceof OlapTable && ((OlapTable) table).enableTso()) { + anyEnableTso = true; + break; + } + } + if (!anyEnableTso) { + return tso; + } + try { + Env env = Env.getCurrentEnv(); + if (env == null || env.getTSOService() == null) { + throw new TransactionCommitFailedException("failed to get TSO for txn " + + transactionId + ": TSO service is unavailable"); + } + long fetched = env.getTSOService().getTSO(); + if (fetched <= 0) { + throw new TransactionCommitFailedException("failed to get TSO for txn " + + transactionId + ", fetched=" + fetched); + } + return fetched; + } catch (RuntimeException e) { + throw new TransactionCommitFailedException("failed to get TSO for txn " + transactionId, e); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/ProcServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/ProcServiceTest.java index 15493325314187..ba19e83781b601 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/ProcServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/ProcServiceTest.java @@ -198,8 +198,6 @@ public void testTabletProc() throws AnalysisException { int backendIdIdx = replicasTitles.indexOf("BackendId"); int versionIdx = replicasTitles.indexOf("Version"); int lastSuccessVersionIdx = replicasTitles.indexOf("LstSuccessVersion"); - int replicasBinlogSizeIdx = replicasTitles.indexOf("BinlogSize"); - int replicasBinlogFileNumIdx = replicasTitles.indexOf("BinlogFileNum"); long tabletId = 10001L; long backendId = 10002L; @@ -228,8 +226,6 @@ public void testTabletProc() throws AnalysisException { Mockito.when(replica.getBackendIdWithoutException()).thenReturn(backendId); Mockito.when(replica.getVersion()).thenReturn(101L); Mockito.when(replica.getLastSuccessVersion()).thenReturn(100L); - Mockito.when(replica.getBinlogSize()).thenReturn(8192L); - Mockito.when(replica.getBinlogFileNum()).thenReturn(5L); try (MockedStatic mockedEnvStatic = Mockito.mockStatic(Env.class)) { mockedEnvStatic.when(Env::getCurrentSystemInfo).thenReturn(systemInfoService); @@ -246,8 +242,6 @@ public void testTabletProc() throws AnalysisException { Assert.assertEquals("10002", row.get(backendIdIdx)); Assert.assertEquals("101", row.get(versionIdx)); Assert.assertEquals("100", row.get(lastSuccessVersionIdx)); - Assert.assertEquals("8192", row.get(replicasBinlogSizeIdx)); - Assert.assertEquals("5", row.get(replicasBinlogFileNumIdx)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java index 9feaba171e5231..45439eac24ee93 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java @@ -152,7 +152,7 @@ private static void bumpPartitionsAndReplicas(OlapTable table, long newVersion) long ts = System.currentTimeMillis(); partition.setVisibleVersionAndTime(newVersion, ts, ts); partition.setNextVersion(newVersion + 1); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.updateVersion(newVersion); @@ -279,13 +279,7 @@ public void testPartitionOffsetAndScanRangeVersion() throws Exception { // The history-partition scan now reads the base table directly; scan range version // is the partition visible version (the legacy stream-tso override was removed). - Map tabletIdToPartitionId = new java.util.HashMap<>(); - for (Partition partition : baseTable.getPartitions()) { - MaterializedIndex baseIndex = partition.getIndex(baseTable.getBaseIndexId()); - for (Tablet tablet : baseIndex.getTablets()) { - tabletIdToPartitionId.put(tablet.getId(), partition.getId()); - } - } + Map tabletIdToPartitionId = buildTabletIdToPartitionId(baseTable); List locations = scanNode.getScanRangeLocations(Long.MAX_VALUE); Assertions.assertFalse(locations.isEmpty()); @@ -350,13 +344,7 @@ public void testIncrementalScanRangeUsesPartitionOffsetAsStartTSO() throws Excep Assertions.assertFalse(scanNodes.isEmpty()); TBinlogScanType expectedScanType = BaseTableStream.StreamScanType.toThrift(stream.getStreamScanType()); - Map tabletIdToPartitionId = new java.util.HashMap<>(); - for (Partition partition : baseTable.getPartitions()) { - MaterializedIndex baseIndex = partition.getIndex(baseTable.getBaseIndexId()); - for (Tablet tablet : baseIndex.getTablets()) { - tabletIdToPartitionId.put(tablet.getId(), partition.getId()); - } - } + Map tabletIdToPartitionId = buildTabletIdToPartitionId(baseTable); boolean assertedAtLeastOne = false; for (OlapScanNode scanNode : scanNodes) { @@ -440,13 +428,7 @@ public void testIncrementalScanStartTsoAdvancesAfterOffsetCommit() throws Except List scanNodes2 = new ArrayList<>(); collectOlapScanNodes(fragment2.getPlanRoot(), scanNodes2); - Map tabletIdToPartitionId = new java.util.HashMap<>(); - for (Partition partition : baseTable.getPartitions()) { - MaterializedIndex baseIndex = partition.getIndex(baseTable.getBaseIndexId()); - for (Tablet tablet : baseIndex.getTablets()) { - tabletIdToPartitionId.put(tablet.getId(), partition.getId()); - } - } + Map tabletIdToPartitionId = buildTabletIdToPartitionId(baseTable); boolean assertedAtLeastOne = false; for (OlapScanNode scanNode : scanNodes2) { if (!(scanNode.getOlapTable() instanceof RowBinlogTableWrapper)) { @@ -555,6 +537,18 @@ public void testDupTimeTravelQualifiedColumnCanBind() { + "from test_stream.tbl_dup_stream_base for version as of 1001")); } + private Map buildTabletIdToPartitionId(OlapTable baseTable) { + Map tabletIdToPartitionId = new java.util.HashMap<>(); + for (Partition partition : baseTable.getPartitions()) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { + for (Tablet tablet : index.getTablets()) { + tabletIdToPartitionId.put(tablet.getId(), partition.getId()); + } + } + } + return tabletIdToPartitionId; + } + private void collectOlapScanNodes(PlanNode node, List result) { if (node instanceof OlapScanNode) { result.add((OlapScanNode) node); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommandTest.java index 9d3c335721fb0c..9ca03e145466e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletIdCommandTest.java @@ -75,6 +75,8 @@ public void testValidate() throws Exception { runBefore(CatalogMocker.TEST_DB_NAME, true); ShowTabletIdCommand command = new ShowTabletIdCommand(CatalogMocker.TEST_TBL_ID); Assertions.assertDoesNotThrow(() -> command.validate(ctx)); + Assertions.assertTrue(command.getMetaData().getColumns().stream() + .anyMatch(column -> column.getName().equals("IsRowBinlog"))); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java index 17613cd057ce02..e24e6bdd6293b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowTabletsFromTableCommandTest.java @@ -84,6 +84,8 @@ public void testValidateWithPrivilege() throws Exception { ShowTabletsFromTableCommand command = new ShowTabletsFromTableCommand(tableNameInfo, partitionNamesInfo, whereClauseNormal, orderKeysNormal, 5, 0); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertTrue(command.getMetaData().getColumns().stream() + .anyMatch(column -> column.getName().equals("IsRowBinlog"))); // where clause error Expression error = new UnboundSlot("error"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommandTableStreamTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommandTableStreamTest.java index 05103aab524123..5ce737402285a7 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommandTableStreamTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommandTableStreamTest.java @@ -270,7 +270,7 @@ public void testInsertProducedStreamUpdateNextEqualsPartitionTsoAndPrevFromHisto long newVer = 5000L + partition.getId() % 1000; partition.setVisibleVersionAndTime(newVer, newVer, newVer); partition.setNextVersion(newVer + 1); - for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE_WITH_ROW_BINLOG)) { for (Tablet tablet : index.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.updateVersion(newVer); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java index ededa002296bcb..ad9bd11a3ceb78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; -import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; @@ -120,7 +119,7 @@ public void setUp() throws AnalysisException { indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false, - storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, null); + storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5); // drop dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false); @@ -172,26 +171,19 @@ public void toThriftTest() throws Exception { Assert.assertEquals(createReplicaTask.getSignature(), request.getSignature()); Assert.assertNotNull(request.getCreateTabletReq()); - // create with row binlog schema + // create with row binlog tablet BinlogConfig binlogConfig = BinlogTestUtils.newTestRowBinlogConfig(true, false); - List rowBinlogColumns = new LinkedList<>(); - rowBinlogColumns.add(new Column("k1", ScalarType.createType(PrimitiveType.INT), true, null, "1", "")); - rowBinlogColumns.add(new Column("v1", ScalarType.createType(PrimitiveType.INT), false, - AggregateType.NONE, "1", "")); - MaterializedIndexMeta rowBinlogMeta = new MaterializedIndexMeta(9999L, rowBinlogColumns, 1, 1, - (short) 1, TStorageType.COLUMN, KeysType.DUP_KEYS, null); - rowBinlogMeta.initSchemaColumnUniqueId(); - - AgentTask createWithRowBinlog = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, + CreateReplicaTask createWithRowBinlog = new CreateReplicaTask(backendId1, dbId, tableId, partitionId, indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType, TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null, TCompressionType.LZ4F, false, "", false, false, "", 0, 0, 0, 0, 0, false, binlogConfig, null, objectPool, rowStorePageSize, false, storagePageSize, - TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5, rowBinlogMeta); + TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5); + createWithRowBinlog.setIsRowBinlogTablet(true); TAgentTaskRequest requestWithRowBinlog = (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, createWithRowBinlog); Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq()); - Assert.assertNotNull(requestWithRowBinlog.getCreateTabletReq().getRowBinlogSchema()); + Assert.assertTrue(requestWithRowBinlog.getCreateTabletReq().isIsRowBinlogTablet()); // drop TAgentTaskRequest request2 = (TAgentTaskRequest) toAgentTaskRequest.invoke(agentBatchTask, dropTask); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index e5c458132a94e1..1d7764bc90f6b1 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -533,6 +533,7 @@ message TxnInfoPB { // The cluster_id of the compute cluster that performed the load // Set by FE during begin_txn, used by commit_txn for compaction read-write separation optional string load_cluster_id = 22; + optional int64 commit_tso = 23; } // For check txn conflict and txn timeout @@ -1021,6 +1022,7 @@ message CommitTxnRequest { repeated SubTxnInfo sub_txn_infos = 10; optional bool enable_txn_lazy_commit = 11; optional string request_ip = 12; + optional int64 commit_tso = 13; } message SubTxnInfo { @@ -1358,6 +1360,22 @@ message CreateRowsetResponse { optional doris.RowsetMetaCloudPB existed_rowset_meta = 2; } +message CreateRowsetsRequest { + optional string cloud_unique_id = 1; // For auth + // Currently only supports data rowset + row-binlog rowset. + optional doris.RowsetMetaCloudPB rowset_meta = 2; + optional doris.RowsetMetaCloudPB attach_row_binlog = 3; + optional int64 txn_id = 4; + optional string tablet_job_id = 5; + optional string request_ip = 6; +} + +message CreateRowsetsResponse { + optional MetaServiceResponseStatus status = 1; + optional doris.RowsetMetaCloudPB existed_rowset_meta = 2; + optional doris.RowsetMetaCloudPB existed_attach_row_binlog = 3; +} + message GetRowsetRequest { enum SchemaOp { FILL_WITH_DICT = 0; // fill rowset schema with SchemaCloudDictionary value @@ -2315,7 +2333,9 @@ service MetaService { rpc get_tablet(GetTabletRequest) returns (GetTabletResponse); rpc prepare_rowset(CreateRowsetRequest) returns (CreateRowsetResponse); rpc commit_rowset(CreateRowsetRequest) returns (CreateRowsetResponse); + rpc commit_rowsets(CreateRowsetsRequest) returns (CreateRowsetsResponse); rpc update_tmp_rowset(CreateRowsetRequest) returns (CreateRowsetResponse); + rpc update_tmp_rowsets(CreateRowsetsRequest) returns (CreateRowsetsResponse); rpc get_rowset(GetRowsetRequest) returns (GetRowsetResponse); rpc get_schema_dict(GetSchemaDictRequest) returns (GetSchemaDictResponse); rpc prepare_index(IndexRequest) returns (IndexResponse); diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 92fc3c0a1a35a1..b82bd724f2dd53 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -123,6 +123,5 @@ message POlapTableSchemaParam { optional UniqueKeyUpdateModePB unique_key_update_mode = 15 [default = UPSERT]; optional int32 sequence_map_col_unique_id = 16 [default = -1]; optional PartialUpdateNewRowPolicyPB partial_update_new_key_policy = 17 [default = APPEND]; - optional POlapTableIndexSchema row_binlog_index_schema = 18; + repeated POlapTableIndexSchema row_binlog_index_schemas = 18; }; - diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 04927146762a16..92a53936a892c3 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -101,6 +101,7 @@ message PResetGlobalRfResult { message PTabletWithPartition { required int64 partition_id = 1; required int64 tablet_id = 2; + optional int64 binlog_tablet_id = 3; } message PRandomBucketPartitionParam { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 7f1c62a446100b..7e8de5184afa24 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -707,9 +707,7 @@ message TabletMetaPB { optional int64 storage_policy_id = 25; optional PUniqueId cooldown_meta_id = 26; optional BinlogConfigPB binlog_config = 27; - optional TabletSchemaPB row_binlog_schema = 42; - repeated RowsetMetaPB row_binlog_rs_metas = 43; - optional int32 row_binlog_schema_hash = 44; + optional int64 binlog_tablet_id = 42; optional string compaction_policy = 28 [default = "size_based"]; optional int64 time_series_compaction_goal_size_mbytes = 29 [default = 1024]; optional int64 time_series_compaction_file_count_threshold = 30 [default = 1000]; @@ -718,6 +716,7 @@ message TabletMetaPB { optional int64 time_series_compaction_level_threshold = 33 [default = 1]; optional EncryptionAlgorithmPB encryption_algorithm = 34; optional int32 vertical_compaction_num_columns_per_group = 35 [default = 5]; + optional bool is_row_binlog_tablet = 36 [default = false]; // For cloud optional int64 index_id = 1000; @@ -766,9 +765,6 @@ message TabletMetaCloudPB { optional int64 storage_policy_id = 28; optional PUniqueId cooldown_meta_id = 29; optional BinlogConfigPB binlog_config = 30; - optional TabletSchemaCloudPB row_binlog_schema = 42; - repeated RowsetMetaCloudPB row_binlog_rs_metas = 43; - optional int32 row_binlog_schema_hash = 44; optional string compaction_policy = 31 [default = "size_based"]; optional int64 time_series_compaction_goal_size_mbytes = 32 [default = 1024]; optional int64 time_series_compaction_file_count_threshold = 33 [default = 1000]; @@ -779,6 +775,7 @@ message TabletMetaCloudPB { optional int64 time_series_compaction_level_threshold = 38 [default = 1]; optional EncryptionAlgorithmPB encryption_algorithm = 39; optional int32 vertical_compaction_num_columns_per_group = 40 [default = 5]; + optional bool is_row_binlog_tablet = 41 [default = false]; // Use for selectdb-cloud optional string table_name = 101; @@ -796,7 +793,6 @@ message DeleteBitmapPB { repeated int64 versions = 3; // Serialized roaring bitmaps indexed with {rowset_id, segment_id, version} repeated bytes segment_delete_bitmaps = 4; - repeated bool is_binlog_delvec = 5; } message BinlogMetaEntryPB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index e78ab617a7986a..96c92ec6823c9c 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -75,14 +75,6 @@ enum TEncryptionAlgorithm { SM4 = 2 } -// Bit flags used by clone/snapshot to describe which tablet files must be copied. -// Request fields use i32 bitmasks of TTabletCopyType values. -enum TTabletCopyType { - DATA = 1, - ROW_BINLOG = 2, - CCR_BINLOG = 4 -} - enum TTabletType { TABLET_TYPE_DISK = 0, TABLET_TYPE_MEMORY = 1 @@ -254,7 +246,7 @@ struct TCreateTabletReq { 29: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2 30: optional TEncryptionAlgorithm tde_algorithm 31: optional i32 vertical_compaction_num_columns_per_group = 5 - 32: optional TTabletSchema row_binlog_schema + 32: optional bool is_row_binlog_tablet = false // For cloud 1000: optional bool is_in_memory = false @@ -387,7 +379,6 @@ struct TCloneReq { 11: optional Types.TReplicaId replica_id = 0 12: optional i64 partition_id 13: optional i64 table_id = -1 - 14: optional i32 copy_type = 5 // bitmask of TTabletCopyType, DATA | CCR_BINLOG by default } struct TCompactionReq { @@ -467,7 +458,6 @@ struct TSnapshotRequest { 12: optional Types.TVersion end_version 13: optional bool is_copy_binlog 14: optional Types.TTabletId ref_tablet_id - 15: optional i32 copy_type = 1 // bitmask of TTabletCopyType, DATA by default } struct TReleaseSnapshotRequest { diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 3867cec245f12d..19d80620702f5c 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -40,8 +40,6 @@ struct TTabletStat { 9: optional i64 local_segment_size = 0 // .dat 10: optional i64 remote_index_size = 0 // .idx 11: optional i64 remote_segment_size = 0 // .dat - 12: optional i64 binlog_size = 0 // __row_binlog/xxx.dat - 13: optional i64 binlog_file_num = 0 } struct TTabletStatResult { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index ca20687d9a90ed..1071cd201d3bfc 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -365,12 +365,14 @@ struct TOlapTableSchemaParam { 14: optional Types.TUniqueKeyUpdateMode unique_key_update_mode 15: optional i32 sequence_map_col_unique_id = -1 16: optional TPartialUpdateNewRowPolicy partial_update_new_key_policy - 17: optional TOlapTableIndexSchema row_binlog_index_schema + 17: optional list row_binlog_index_schemas } struct TTabletLocation { 1: required i64 tablet_id 2: required list node_ids + // used to write binlog tablet by base tablet with the same bucket idx + 3: optional i64 base_tablet_id } struct TOlapTableLocationParam { diff --git a/gensrc/thrift/MasterService.thrift b/gensrc/thrift/MasterService.thrift index a468ab9d6a88a4..da65873eb6ee93 100644 --- a/gensrc/thrift/MasterService.thrift +++ b/gensrc/thrift/MasterService.thrift @@ -51,8 +51,6 @@ struct TTabletInfo { 23: optional i64 local_segment_size = 0 // .dat 24: optional i64 remote_index_size = 0 // .idx 25: optional i64 remote_segment_size = 0 // .dat - 26: optional i64 binlog_size = 0 // __row_binlog/xxx.dat - 27: optional i64 binlog_file_num = 0 // For cloud 1000: optional bool is_persistent diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 578ff2fca799fb..778df23e5ccf3e 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -636,6 +636,7 @@ struct TFoldConstantParams { struct TTabletWithPartition { 1: required i64 partition_id 2: required i64 tablet_id + 3: optional i64 binlog_tablet_id } struct TFetchDataResult { @@ -750,6 +751,7 @@ struct TPipelineFragmentParams { // For cloud 1000: optional bool is_mow_table; + 1001: optional bool enable_tso; } // pull up runtime filter info from instance level to BE level diff --git a/regression-test/data/row_binlog_p0/test_cloud_row_binlog_compaction.out b/regression-test/data/row_binlog_p0/test_cloud_row_binlog_compaction.out new file mode 100644 index 00000000000000..a2d5cf742a418c --- /dev/null +++ b/regression-test/data/row_binlog_p0/test_cloud_row_binlog_compaction.out @@ -0,0 +1,38 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cloud_dup_binlog_compaction -- +0 1 10 a +0 1 11 a1 +0 2 20 b +0 3 30 c + +-- !cloud_mow_historical_binlog_compaction -- +0 1 10 a \N \N +1 1 11 a1 10 a +0 2 20 b \N \N +1 2 21 b1 20 b +0 3 30 c \N \N +1 1 12 a2 11 a1 +1 1 13 a3 12 a2 +1 2 22 b2 21 b1 +2 3 30 c 30 c + +-- !cloud_mow_seq_historical_binlog_compaction -- +0 1 100 a1 \N \N +1 1 300 a3 100 a1 +0 2 200 b1 \N \N +2 2 200 b1 200 b1 + +-- !cloud_mow_seq_historical_binlog_compaction_skip_delete_bitmap -- +0 1 100 a1 \N \N +1 1 300 a3 100 a1 +0 2 200 b1 \N \N +2 2 200 b1 200 b1 + +-- !cloud_mow_historical_long_chain_binlog_compaction -- +0 1 100 a0 \N \N +1 1 110 a1 100 a0 +1 1 120 a2 110 a1 +1 1 130 a3 120 a2 +1 1 140 a4 130 a3 +0 2 200 b0 \N \N +2 2 200 b0 200 b0 diff --git a/regression-test/data/row_binlog_p0/test_cloud_row_binlog_publish_conflict.out b/regression-test/data/row_binlog_p0/test_cloud_row_binlog_publish_conflict.out new file mode 100644 index 00000000000000..74e268f25b6d6e --- /dev/null +++ b/regression-test/data/row_binlog_p0/test_cloud_row_binlog_publish_conflict.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !row_binlog_publish_conflict -- +0 1 1 1 10 10 \N \N +0 2 2 2 20 20 \N \N +0 3 3 3 30 30 \N \N +1 2 2 2 20 200 20 20 +1 3 3 3 30 300 30 30 +1 2 2 2 2200 200 20 200 + +-- !row_binlog_publish_conflict_skip_bitmap -- +0 1 1 1 10 10 \N \N +0 2 2 2 20 20 \N \N +0 3 3 3 30 30 \N \N +1 2 2 2 20 200 20 20 +1 3 3 3 30 300 30 30 +1 2 2 2 2200 20 20 20 +1 2 2 2 2200 200 20 200 + +-- !row_binlog_publish_conflict_upsert -- +0 2 2 2 20 20 \N \N +0 3 3 3 30 30 \N \N +1 2 2 2 20 200 20 20 +1 3 3 3 30 300 30 30 +1 2 2 2 2200 200 20 200 +1 2 2 2 2000 2000 2200 200 +1 3 3 3 3000 3000 30 300 +1 2 2 2 2222 2222 2000 2000 + +-- !row_binlog_publish_conflict_upsert_skip_bitmap -- +0 2 2 2 20 20 \N \N +0 3 3 3 30 30 \N \N +1 2 2 2 20 200 20 20 +1 3 3 3 30 300 30 30 +1 2 2 2 2200 20 20 20 +1 2 2 2 2200 200 20 200 +1 2 2 2 2000 2000 2200 200 +1 3 3 3 3000 3000 30 300 +1 2 2 2 2222 2222 2200 200 +1 2 2 2 2222 2222 2000 2000 + diff --git a/regression-test/plugins/plugin_compaction.groovy b/regression-test/plugins/plugin_compaction.groovy index 67c16720204a34..6dbab669bfa7fd 100644 --- a/regression-test/plugins/plugin_compaction.groovy +++ b/regression-test/plugins/plugin_compaction.groovy @@ -20,20 +20,24 @@ import java.util.concurrent.TimeUnit import org.awaitility.Awaitility; Suite.metaClass.be_get_compaction_status{ String ip, String port, String tablet_id /* param */-> - return curl("GET", String.format("http://%s:%s/api/compaction/run_status?tablet_id=%s", ip, port, tablet_id)) + return curl("GET", String.format("http://%s:%s/api/compaction/run_status?tablet_id=%s", ip, port, tablet_id), + null, 10, context.config.feHttpUser, context.config.feHttpPassword) } Suite.metaClass.be_get_overall_compaction_status{ String ip, String port /* param */-> - return curl("GET", String.format("http://%s:%s/api/compaction/run_status", ip, port)) + return curl("GET", String.format("http://%s:%s/api/compaction/run_status", ip, port), + null, 10, context.config.feHttpUser, context.config.feHttpPassword) } Suite.metaClass.be_show_tablet_status{ String ip, String port, String tablet_id /* param */-> - return curl("GET", String.format("http://%s:%s/api/compaction/show?tablet_id=%s", ip, port, tablet_id)) + return curl("GET", String.format("http://%s:%s/api/compaction/show?tablet_id=%s", ip, port, tablet_id), + null, 10, context.config.feHttpUser, context.config.feHttpPassword) } Suite.metaClass._be_run_compaction = { String ip, String port, String tablet_id, String compact_type -> return curl("POST", String.format("http://%s:%s/api/compaction/run?tablet_id=%s&compact_type=%s", - ip, port, tablet_id, compact_type)) + ip, port, tablet_id, compact_type), null, 10, + context.config.feHttpUser, context.config.feHttpPassword) } Suite.metaClass.be_run_base_compaction = { String ip, String port, String tablet_id /* param */-> @@ -52,19 +56,15 @@ Suite.metaClass.be_run_full_compaction = { String ip, String port, String tablet return _be_run_compaction(ip, port, tablet_id, "full") } -Suite.metaClass.be_run_binlog_compaction = { String ip, String port, String tablet_id /* param */-> - return _be_run_compaction(ip, port, tablet_id, "binlog") -} - Suite.metaClass.be_run_full_compaction_by_table_id = { String ip, String port, String table_id /* param */-> - return curl("POST", String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full", ip, port, table_id)) + return curl("POST", String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full", ip, port, table_id), + null, 10, context.config.feHttpUser, context.config.feHttpPassword) } logger.info("Added 'be_run_full_compaction' function to Suite") -logger.info("Added 'be_run_binlog_compaction' function to Suite") Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compaction_type, int timeout_seconds=300, String[] ignored_errors=[] -> - if (!(compaction_type in ["cumulative", "base", "full", "binlog"])) { - throw new IllegalArgumentException("invalid compaction type: ${compaction_type}, supported types: cumulative, base, full, binlog") + if (!(compaction_type in ["cumulative", "base", "full"])) { + throw new IllegalArgumentException("invalid compaction type: ${compaction_type}, supported types: cumulative, base, full") } def backendId_to_backendIP = [:] @@ -102,9 +102,6 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac case "full": (exit_code, stdout, stderr) = be_run_full_compaction(be_host, be_port, tablet.TabletId) break - case "binlog": - (exit_code, stdout, stderr) = be_run_binlog_compaction(be_host, be_port, tablet.TabletId) - break } assert exit_code == 0: "trigger compaction failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}" def trigger_status = parseJson(stdout.trim()) diff --git a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy index 80d98d13f70678..2cba9683ceb1fd 100644 --- a/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy +++ b/regression-test/suites/row_binlog_p0/test_binlog_changes_syntax.groovy @@ -16,9 +16,6 @@ // under the License. suite("test_binlog_changes_syntax", "nonConcurrent") { - if (isCloudMode()) { - return - } sql "DROP DATABASE IF EXISTS test_binlog_changes_syntax_db" sql "CREATE DATABASE test_binlog_changes_syntax_db" sql "USE test_binlog_changes_syntax_db" diff --git a/regression-test/suites/row_binlog_p0/test_binlog_compaction.groovy b/regression-test/suites/row_binlog_p0/test_binlog_compaction.groovy index 7be09e902f262f..42faa9eabdfdca 100644 --- a/regression-test/suites/row_binlog_p0/test_binlog_compaction.groovy +++ b/regression-test/suites/row_binlog_p0/test_binlog_compaction.groovy @@ -59,7 +59,7 @@ suite("test_binlog_compaction", "nonConcurrent") { """ // Round 1: Level0 [0-1], [2-2], [3-3] -> Level1 [0-3]. - trigger_and_wait_compaction("test_binlog_compaction_dup", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_dup", "cumulative") qt_dup_binlog_compaction """ SELECT __DORIS_BINLOG_OP__ AS op, @@ -100,7 +100,7 @@ suite("test_binlog_compaction", "nonConcurrent") { (1, 11, 'a1') """ // Round 1: Level0 [0-1], [2-2], [3-3] -> Level1 [0-3]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") sql """ INSERT INTO test_binlog_compaction_mow_historical VALUES @@ -111,9 +111,9 @@ suite("test_binlog_compaction", "nonConcurrent") { (3, 30, 'c') """ // Round 2: Level0 [4-4], [5-5] -> Level1 [4-5]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") // Round 3: Level1 [0-3], [4-5] -> Level2 [0-5]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") sql """ INSERT INTO test_binlog_compaction_mow_historical VALUES @@ -124,7 +124,7 @@ suite("test_binlog_compaction", "nonConcurrent") { (1, 13, 'a3') """ // Round 4: Level0 [6-6], [7-7] -> Level1 [6-7]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") sql """ INSERT INTO test_binlog_compaction_mow_historical VALUES @@ -132,11 +132,11 @@ suite("test_binlog_compaction", "nonConcurrent") { """ sql "DELETE FROM test_binlog_compaction_mow_historical WHERE k1 = 3" // Round 5: Level0 [8-8], [9-9] -> Level1 [8-9]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") // Round 6: Level1 [6-7], [8-9] -> Level2 [6-9]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") // Round 7: Level2 [0-5], [6-9] -> [0-9]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical", "cumulative") qt_mow_historical_binlog_compaction """ SELECT __DORIS_BINLOG_OP__ AS op, @@ -185,7 +185,7 @@ suite("test_binlog_compaction", "nonConcurrent") { (1, 90, 'a0', 1) """ // Round 1: Level0 [0-1], [2-2], [3-3] -> Level1 [0-3]. - trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "cumulative") sql """ INSERT INTO test_binlog_compaction_mow_seq_historical(k1, v1, v2, __DORIS_SEQUENCE_COL__) @@ -194,9 +194,9 @@ suite("test_binlog_compaction", "nonConcurrent") { """ sql "DELETE FROM test_binlog_compaction_mow_seq_historical WHERE k1 = 2" // Round 2: Level0 [4-4], [5-5] -> Level1 [4-5]. - trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "cumulative") // Round 3: Level1 [0-3], [4-5] -> Level2 [0-5]. - trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_seq_historical", "cumulative") qt_mow_seq_historical_binlog_compaction """ SELECT __DORIS_BINLOG_OP__ AS op, @@ -243,7 +243,7 @@ suite("test_binlog_compaction", "nonConcurrent") { (1, 120, 'a2') """ // Round 1: Level0 [0-1], [2-2], [3-3], [4-4] -> Level1 [0-4]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "cumulative") sql """ INSERT INTO test_binlog_compaction_mow_historical_long_chain VALUES @@ -255,9 +255,9 @@ suite("test_binlog_compaction", "nonConcurrent") { """ sql "DELETE FROM test_binlog_compaction_mow_historical_long_chain WHERE k1 = 2" // Round 2: Level0 [5-5], [6-6], [7-7] -> Level1 [5-7]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "cumulative") // Round 3: Level1 [0-4], [5-7] -> Level2 [0-7]. - trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "binlog") + trigger_and_wait_compaction("test_binlog_compaction_mow_historical_long_chain", "cumulative") qt_mow_historical_long_chain_binlog_compaction """ SELECT __DORIS_BINLOG_OP__ AS op, diff --git a/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_compaction.groovy b/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_compaction.groovy new file mode 100644 index 00000000000000..97c916fb54f299 --- /dev/null +++ b/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_compaction.groovy @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cloud_row_binlog_compaction", "nonConcurrent") { + if (!isCloudMode()) { + return + } + + setBeConfigTemporary([ + binlog_compaction_goal_size_mbytes: 0, + binlog_compaction_file_count_threshold: 2, + binlog_compaction_wait_timesec_after_visible: 0, + binlog_compaction_time_threshold_seconds: 86400 + ]) { + // Case 1: DUP_KEYS inserts. + sql "DROP TABLE IF EXISTS test_cloud_binlog_compaction_dup FORCE" + sql """ + CREATE TABLE test_cloud_binlog_compaction_dup ( + k1 INT, + v1 INT, + v2 STRING + ) + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW" + ) + """ + + sql """ + INSERT INTO test_cloud_binlog_compaction_dup VALUES + (1, 10, 'a'), + (2, 20, 'b') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_dup VALUES + (1, 11, 'a1'), + (3, 30, 'c') + """ + + // Round 1: Level0 [2-2], [3-3] -> Level1 [2-3]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_dup", "cumulative") + + qt_cloud_dup_binlog_compaction """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + v1, + v2 + FROM binlog("table" = "test_cloud_binlog_compaction_dup") + """ + + // Case 2: MOW inserts, historical values, and delete. + sql "DROP TABLE IF EXISTS test_cloud_binlog_compaction_mow_historical FORCE" + sql """ + CREATE TABLE test_cloud_binlog_compaction_mow_historical ( + k1 INT, + v1 INT, + v2 STRING + ) + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "light_schema_change" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (1, 10, 'a'), + (2, 20, 'b') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (1, 11, 'a1') + """ + // Round 1: Level0 [2-2], [3-3] -> Level1 [2-3]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (2, 21, 'b1') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (3, 30, 'c') + """ + // Round 2: Level0 [4-4], [5-5] -> Level1 [4-5]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + // Round 3: Level1 [2-3], [4-5] -> Level2 [2-5]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (1, 12, 'a2') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (1, 13, 'a3') + """ + // Round 4: Level0 [6-6], [7-7] -> Level1 [6-7]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical VALUES + (2, 22, 'b2') + """ + sql "DELETE FROM test_cloud_binlog_compaction_mow_historical WHERE k1 = 3" + // Round 5: Level0 [8-8], [9-9] -> Level1 [8-9]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + // Round 6: Level1 [6-7], [8-9] -> Level2 [6-9]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical", "cumulative") + + qt_cloud_mow_historical_binlog_compaction """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_cloud_binlog_compaction_mow_historical") + """ + + // Case 3: MOW sequence inserts, historical values, and delete. + sql "DROP TABLE IF EXISTS test_cloud_binlog_compaction_mow_seq_historical FORCE" + sql """ + CREATE TABLE test_cloud_binlog_compaction_mow_seq_historical ( + k1 INT, + v1 INT, + v2 STRING + ) + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "light_schema_change" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + + sql """ALTER TABLE test_cloud_binlog_compaction_mow_seq_historical + ENABLE FEATURE "SEQUENCE_LOAD" + WITH PROPERTIES ("function_column.sequence_type" = "int")""" + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_seq_historical(k1, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES + (1, 100, 'a1', 2), + (2, 200, 'b1', 1) + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_seq_historical(k1, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES + (1, 90, 'a0', 1) + """ + // Round 1: Level0 [2-2], [3-3] -> Level1 [2-3]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_seq_historical", + "cumulative") + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_seq_historical(k1, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES + (1, 300, 'a3', 3) + """ + sql "DELETE FROM test_cloud_binlog_compaction_mow_seq_historical WHERE k1 = 2" + // Round 2: Level0 [4-4], [5-5] -> Level1 [4-5]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_seq_historical", + "cumulative") + // Round 3: Level1 [2-3], [4-5] -> Level2 [2-5]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_seq_historical", + "cumulative") + + sql "SET skip_delete_bitmap = false" + qt_cloud_mow_seq_historical_binlog_compaction """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_cloud_binlog_compaction_mow_seq_historical") + """ + + sql "SET skip_delete_bitmap = true" + qt_cloud_mow_seq_historical_binlog_compaction_skip_delete_bitmap """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_cloud_binlog_compaction_mow_seq_historical") + """ + sql "SET skip_delete_bitmap = false" + + // Case 4: MOW long-chain inserts, historical values, and delete. + sql "DROP TABLE IF EXISTS test_cloud_binlog_compaction_mow_historical_long_chain FORCE" + sql """ + CREATE TABLE test_cloud_binlog_compaction_mow_historical_long_chain ( + k1 INT, + v1 INT, + v2 STRING + ) + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "light_schema_change" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical_long_chain VALUES + (1, 100, 'a0'), + (2, 200, 'b0') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical_long_chain VALUES + (1, 110, 'a1') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical_long_chain VALUES + (1, 120, 'a2') + """ + // Round 1: Level0 [2-2], [3-3], [4-4] -> Level1 [2-4]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical_long_chain", + "cumulative") + + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical_long_chain VALUES + (1, 130, 'a3') + """ + sql """ + INSERT INTO test_cloud_binlog_compaction_mow_historical_long_chain VALUES + (1, 140, 'a4') + """ + sql "DELETE FROM test_cloud_binlog_compaction_mow_historical_long_chain WHERE k1 = 2" + // Round 2: Level0 [5-5], [6-6], [7-7] -> Level1 [5-7]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical_long_chain", + "cumulative") + // Round 3: Level1 [2-4], [5-7] -> Level2 [2-7]. + trigger_and_wait_compaction("test_cloud_binlog_compaction_mow_historical_long_chain", + "cumulative") + + qt_cloud_mow_historical_long_chain_binlog_compaction """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_cloud_binlog_compaction_mow_historical_long_chain") + """ + } +} diff --git a/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_publish_conflict.groovy b/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_publish_conflict.groovy new file mode 100644 index 00000000000000..389d8033143a80 --- /dev/null +++ b/regression-test/suites/row_binlog_p0/test_cloud_row_binlog_publish_conflict.groovy @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicReference + +suite("test_cloud_row_binlog_publish_conflict", "nonConcurrent") { + if (!isCloudMode()) { + return + } + + sql "DROP TABLE IF EXISTS test_mow_publish_conflict_with_binlog FORCE" + + sql """ + CREATE TABLE test_mow_publish_conflict_with_binlog ( + k1 INT, + k2 INT, + k3 INT, + v1 INT, + v2 STRING + ) + UNIQUE KEY(k1, k2, k3) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "disable_auto_compaction" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + + sql """ALTER TABLE test_mow_publish_conflict_with_binlog + ENABLE FEATURE "SEQUENCE_LOAD" + WITH PROPERTIES ("function_column.sequence_type" = "int")""" + + sql """ + INSERT INTO test_mow_publish_conflict_with_binlog(k1, k2, k3, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES + (1, 1, 1, 10, '10', 1), + (2, 2, 2, 20, '20', 1), + (3, 3, 3, 30, '30', 1) + """ + sql "sync" + + def block_publish = { token, passToken -> + GetDebugPoint().enableDebugPointForAllFEs( + "CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait", + [token: token, pass_token: passToken]) + } + + def pass_publish = { token -> + GetDebugPoint().enableDebugPointForAllFEs( + "CloudGlobalTransactionMgr.tryCommitLock.enable_spin_wait", + [token: token, pass_token: token]) + } + + AtomicReference txn2Error = new AtomicReference<>() + AtomicReference txn4Error = new AtomicReference<>() + + try { + GetDebugPoint().clearDebugPointsForAllFEs() + block_publish("txn2", "txn1") + + def txn2 = Thread.start { + try { + sql "SET enable_unique_key_partial_update = true" + sql """ + INSERT INTO test_mow_publish_conflict_with_binlog(k1, k2, k3, v1, __DORIS_SEQUENCE_COL__) + VALUES (2, 2, 2, 2200, 3) + """ + } catch (Throwable t) { + txn2Error.set(t) + } + } + + // Wait for txn2 to reach the Cloud commit-lock debug point. + Thread.sleep(10000) + pass_publish("txn1") + + // Commit two partial-update transactions while Cloud commit is blocked. + sql "SET enable_unique_key_partial_update = true" + sql """ + INSERT INTO test_mow_publish_conflict_with_binlog(k1, k2, k3, v2, __DORIS_SEQUENCE_COL__) + VALUES + (2, 2, 2, '200', 2), + (3, 3, 3, '300', 2) + """ + + block_publish("txn1", "txn2") + txn2.join() + if (txn2Error.get() != null) { + throw txn2Error.get() + } + + qt_row_binlog_publish_conflict """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + k2, + k3, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_mow_publish_conflict_with_binlog") + WHERE k1 IN (1, 2, 3) + ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ + """ + + sql "SET skip_delete_bitmap = true" + qt_row_binlog_publish_conflict_skip_bitmap """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + k2, + k3, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_mow_publish_conflict_with_binlog") + WHERE k1 IN (1, 2, 3) + ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ + """ + + // Reset session variable before testing upsert path. + sql "SET skip_delete_bitmap = false" + sql "SET enable_unique_key_partial_update = false" + block_publish("txn4", "txn3") + + def txn4 = Thread.start { + try { + sql """ + INSERT INTO test_mow_publish_conflict_with_binlog(k1, k2, k3, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES (2, 2, 2, 2222, '2222', 5) + """ + } catch (Throwable t) { + txn4Error.set(t) + } + } + + // Wait for txn4 to reach the Cloud commit-lock debug point. + Thread.sleep(10000) + pass_publish("txn3") + + // Upsert (full columns) goes through a different write/publish path than partial-update. + sql """ + INSERT INTO test_mow_publish_conflict_with_binlog(k1, k2, k3, v1, v2, __DORIS_SEQUENCE_COL__) + VALUES + (2, 2, 2, 2000, '2000', 4), + (3, 3, 3, 3000, '3000', 4) + """ + + block_publish("txn3", "txn4") + txn4.join() + if (txn4Error.get() != null) { + throw txn4Error.get() + } + + qt_row_binlog_publish_conflict_upsert """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + k2, + k3, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_mow_publish_conflict_with_binlog") + WHERE k1 IN (2, 3) + ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ + """ + + sql "SET skip_delete_bitmap = true" + qt_row_binlog_publish_conflict_upsert_skip_bitmap """ + SELECT __DORIS_BINLOG_OP__ AS op, + k1, + k2, + k3, + v1, + v2, + __BEFORE__v1__, + __BEFORE__v2__ + FROM binlog("table" = "test_mow_publish_conflict_with_binlog") + WHERE k1 IN (2, 3) + ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ + """ + } finally { + sql "SET skip_delete_bitmap = false" + sql "SET enable_unique_key_partial_update = false" + GetDebugPoint().clearDebugPointsForAllFEs() + } +} diff --git a/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy b/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy index d810420346ba79..c8149e04eaa71d 100644 --- a/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy +++ b/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy @@ -16,10 +16,6 @@ // under the License. suite("test_row_binlog_basic", "nonConcurrent") { - if (isCloudMode()) { - return - } - sql "DROP TABLE IF EXISTS test_dup_with_binlog FORCE" sql "DROP TABLE IF EXISTS test_mow_with_binlog FORCE" sql "DROP TABLE IF EXISTS test_mow_with_before_binlog FORCE" diff --git a/regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy b/regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy index e23f353bb18839..85d196ae1df11d 100644 --- a/regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy +++ b/regression-test/suites/row_binlog_p0/test_row_binlog_multi_segment.groovy @@ -16,10 +16,6 @@ // under the License. suite("test_row_binlog_multi_segment", "nonConcurrent") { - if (isCloudMode()) { - return - } - sql "DROP TABLE IF EXISTS test_mow_multi_segment_with_binlog FORCE" sql """ diff --git a/regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy b/regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy index 19a00aab740b39..b728ba1c187907 100644 --- a/regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy +++ b/regression-test/suites/row_binlog_p0/test_row_binlog_schema_change.groovy @@ -16,10 +16,6 @@ // under the License. suite("test_row_binlog_schema_change", "nonConcurrent") { - if (isCloudMode()) { - return - } - sql "DROP TABLE IF EXISTS test_mow_schema_change_with_binlog FORCE" sql """