From c5c6829b73d8a7d017fdea893a056a92309d32c8 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Fri, 11 Sep 2026 11:39:44 +0800
Subject: [PATCH 1/8] Add Pipe logical backup sink and restore tooling
---
distribution/src/assembly/cli.xml | 2 +
iotdb-client/cli/pom.xml | 5 +
.../apache/iotdb/cli/i18n/CliMessages.java | 57 +
.../apache/iotdb/cli/i18n/CliMessages.java | 57 +
.../tool/pipe/PipeLogicalBackupTool.java | 817 ++++++++++++
.../tool/pipe/PipeLogicalBackupToolTest.java | 166 +++
.../iotdb/db/i18n/DataNodePipeMessages.java | 11 +
.../iotdb/db/i18n/DataNodePipeMessages.java | 11 +
.../PipeDataRegionSinkConstructor.java | 7 +
.../PipeSchemaRegionSinkConstructor.java | 7 +
.../logicalbackup/LogicalBackupSink.java | 591 +++++++++
iotdb-core/node-commons/pom.xml | 4 +
.../commons/i18n/LogicalBackupMessages.java | 151 +++
.../commons/i18n/LogicalBackupMessages.java | 151 +++
.../plugin/builtin/BuiltinPipePlugin.java | 4 +
.../sink/logicalbackup/LogicalBackupSink.java | 24 +
.../config/constant/PipeSinkConstant.java | 49 +
.../LogicalBackupArchiveReader.java | 481 +++++++
.../logicalbackup/LogicalBackupFormat.java | 62 +
.../logicalbackup/LogicalBackupManifest.java | 68 +
.../logicalbackup/LogicalBackupRecord.java | 136 ++
.../LogicalBackupRecordType.java | 54 +
.../LogicalBackupSegmentReader.java | 541 ++++++++
.../logicalbackup/LogicalBackupWriter.java | 1162 +++++++++++++++++
.../LogicalBackupArchiveReaderTest.java | 272 ++++
.../LogicalBackupWriterTest.java | 588 +++++++++
scripts/tools/export-pipe-logical-backup.sh | 21 +
scripts/tools/import-pipe-logical-backup.sh | 21 +
scripts/tools/pipe-logical-backup.sh | 43 +
.../windows/export-pipe-logical-backup.bat | 21 +
.../windows/import-pipe-logical-backup.bat | 21 +
scripts/tools/windows/pipe-logical-backup.bat | 40 +
32 files changed, 5645 insertions(+)
create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
create mode 100644 iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
create mode 100644 iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
create mode 100644 scripts/tools/export-pipe-logical-backup.sh
create mode 100644 scripts/tools/import-pipe-logical-backup.sh
create mode 100644 scripts/tools/pipe-logical-backup.sh
create mode 100644 scripts/tools/windows/export-pipe-logical-backup.bat
create mode 100644 scripts/tools/windows/import-pipe-logical-backup.bat
create mode 100644 scripts/tools/windows/pipe-logical-backup.bat
diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml
index 6f4eed7f7c25..7e60322b4987 100644
--- a/distribution/src/assembly/cli.xml
+++ b/distribution/src/assembly/cli.xml
@@ -54,9 +54,11 @@
*data.*
*schema.*
*tsfile.*
+ *logical-backup*
**/*data.*
**/*schema.*
**/*tsfile.*
+ **/*logical-backup*
0755
diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml
index 5de247a084a0..c9824ef4aec1 100644
--- a/iotdb-client/cli/pom.xml
+++ b/iotdb-client/cli/pom.xml
@@ -124,6 +124,10 @@
commons-cli
commons-cli
+
+ com.google.code.gson
+ gson
+
org.jline
jline
@@ -204,6 +208,7 @@
org.apache.iotdb:iotdb-server
org/apache/iotdb/db/utils/*
+ org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.class
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 0329812f2d4d..77888e65768a 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -96,4 +96,61 @@ private CliMessages() {}
public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "Input time format {} is not supported, ";
public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n";
+ // Pipe logical backup tool
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 =
+ "Logical backup command failed: %s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 =
+ "Unknown logical backup command: %s";
+ public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 =
+ "--output is required for logical backup export";
+ public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 =
+ "Stream %s: %d records, %d committed event groups";
+ public static final String
+ LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B =
+ "Logical backup verified: %d streams, %d records, %d committed event groups";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 =
+ "Logical backup export target must not be inside source";
+ public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 =
+ "Logical backup exported from %s to %s";
+ public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B =
+ "Dry run completed; no data was written";
+ public static final String
+ LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D =
+ "Logical backup import completed: %d event groups, checkpoint %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA =
+ "Logical backup option --%s is required";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 =
+ "Logical backup password environment variable is not set";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 =
+ "Logical backup handshake failed: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
+ "Logical backup request type %d failed: %s";
+ public static final String
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
+ "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
+ public static final String
+ LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 =
+ "pipe-logical-backup ";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
+ "Logical backup archive entry is unsafe: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
+ "Logical backup archive exceeds safety limit";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 =
+ "Logical backup export source must be a directory or manifest: %s";
+ public static final String
+ EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 =
+ "Specify exactly one of --password-stdin and --password-env for logical backup import";
+ public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E =
+ "No password was read from standard input";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D =
+ "Logical backup checkpoint does not match the source or target: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C =
+ "Logical backup checkpoint is invalid: %s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
+ "Unsupported logical backup export format: %s";
+
}
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index 61a14d94d588..530b504b29ce 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -92,4 +92,61 @@ private CliMessages() {}
public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "不支持输入时间格式 {}，";
public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "请输入类似 yyyy-MM-dd\\ HH:mm:ss.SSS 或 yyyy-MM-dd'T'HH:mm:ss.SSS 的格式%n";
+ // Pipe 逻辑备份工具
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 =
+ "逻辑备份命令执行失败：%s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 =
+ "未知的逻辑备份命令：%s";
+ public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 =
+ "逻辑备份导出必须指定 --output";
+ public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 =
+ "流 %s：%d 条记录，%d 个已提交事件组";
+ public static final String
+ LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B =
+ "逻辑备份校验通过：%d 个流，%d 条记录，%d 个已提交事件组";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 =
+ "逻辑备份导出目标不能位于源目录内";
+ public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 =
+ "逻辑备份已从 %s 导出到 %s";
+ public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B =
+ "试运行完成，未写入数据";
+ public static final String
+ LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D =
+ "逻辑备份导入完成：%d 个事件组，checkpoint 为 %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA =
+ "逻辑备份选项 --%s 为必填项";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 =
+ "未设置逻辑备份密码环境变量";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 =
+ "逻辑备份握手失败：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
+ "逻辑备份请求类型 %d 执行失败：%s";
+ public static final String
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
+ "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
+ public static final String
+ LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 =
+ "pipe-logical-backup ";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
+ "逻辑备份归档项路径不安全：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
+ "逻辑备份归档超过安全限制";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 =
+ "逻辑备份导出源必须是目录或 manifest：%s";
+ public static final String
+ EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 =
+ "逻辑备份导入必须且只能指定 --password-stdin 或 --password-env 中的一项";
+ public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E =
+ "未能从标准输入读取密码";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D =
+ "逻辑备份 checkpoint 与源备份或目标实例不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C =
+ "逻辑备份 checkpoint 无效：%s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
+ "不支持的逻辑备份导出格式：%s";
+
}
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
new file mode 100644
index 000000000000..5d80243f3adf
--- /dev/null
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -0,0 +1,817 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool.pipe;
+
+import org.apache.iotdb.cli.i18n.CliMessages;
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.thrift.TException;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */
+public final class PipeLogicalBackupTool {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final String COMMAND_INSPECT = "inspect";
+ private static final String COMMAND_VERIFY = "verify";
+ private static final String COMMAND_EXPORT = "export";
+ private static final String COMMAND_IMPORT = "import";
+ private static final String COMMAND_RESTORE = "restore";
+ private static final String COMMAND_STATS = "stats";
+ private static final String OPTION_INPUT = "input";
+ private static final String OPTION_OUTPUT = "output";
+ private static final String OPTION_RESUME = "resume";
+ private static final String OPTION_FORMAT = "format";
+ private static final String OPTION_DEEP = "deep";
+ // Compatibility aliases for the initial command-line prototype.
+ private static final String OPTION_SOURCE = "source";
+ private static final String OPTION_TARGET = "target";
+ private static final String OPTION_HOST = "host";
+ private static final String OPTION_PORT = "port";
+ private static final String OPTION_USER = "user";
+ private static final String OPTION_PASSWORD_STDIN = "password-stdin";
+ private static final String OPTION_PASSWORD_ENV = "password-env";
+ private static final String OPTION_CHECKPOINT = "checkpoint";
+ private static final String OPTION_DRY_RUN = "dry-run";
+ private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete";
+ private static final String OPTION_JSON = "json";
+ private static final int MAX_ARCHIVE_ENTRIES = 1_000_000;
+ private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024;
+ private static final String CHECKPOINT_FORMAT_NAME =
+ "iotdb-pipe-logical-backup-import-checkpoint";
+ private static final String CHECKPOINT_FORMAT_VERSION = "1.0";
+
+ private PipeLogicalBackupTool() {}
+
+ public static void main(final String[] args) {
+ int exitCode = 0;
+ try {
+ exitCode = run(args);
+ } catch (final Exception e) {
+ System.err.println(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0, e.getMessage()));
+ exitCode = 1;
+ }
+ if (exitCode != 0) {
+ System.exit(exitCode);
+ }
+ }
+
+ static int run(final String[] args) throws Exception {
+ if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) {
+ printUsage();
+ return 0;
+ }
+ final String command = args[0].toLowerCase(java.util.Locale.ROOT);
+ final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length));
+ switch (command) {
+ case COMMAND_INSPECT:
+ return inspect(line);
+ case COMMAND_VERIFY:
+ return verify(line);
+ case COMMAND_EXPORT:
+ return export(line);
+ case COMMAND_IMPORT:
+ case COMMAND_RESTORE:
+ return importBackup(line);
+ case COMMAND_STATS:
+ return inspect(line);
+ default:
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command));
+ }
+ }
+
+ private static CommandLine parse(final String command, final String[] args)
+ throws ParseException {
+ final Options options = new Options();
+ addOption(options, OPTION_INPUT, true, false);
+ addOption(options, OPTION_OUTPUT, true, false);
+ addOption(options, OPTION_RESUME, true, false);
+ addOption(options, OPTION_FORMAT, true, false);
+ addOption(options, OPTION_DEEP, false, false);
+ addOption(options, OPTION_SOURCE, true, false);
+ addOption(options, OPTION_TARGET, true, false);
+ addOption(options, OPTION_HOST, true, false);
+ addOption(options, OPTION_PORT, true, false);
+ addOption(options, OPTION_USER, true, false);
+ addOption(options, OPTION_PASSWORD_STDIN, false, false);
+ addOption(options, OPTION_PASSWORD_ENV, true, false);
+ addOption(options, OPTION_CHECKPOINT, true, false);
+ addOption(options, OPTION_DRY_RUN, false, false);
+ addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
+ addOption(options, OPTION_JSON, false, false);
+ final CommandLine line = new DefaultParser().parse(options, args);
+ final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
+ if (input == null || input.isBlank()) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
+ }
+ final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET);
+ if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) {
+ throw new ParseException(
+ CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1);
+ }
+ final String format = line.getOptionValue(OPTION_FORMAT, "binary");
+ if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format));
+ }
+ return line;
+ }
+
+ private static void addOption(
+ final Options options, final String name, final boolean hasArg, final boolean required) {
+ options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build());
+ }
+
+ private static int inspect(final CommandLine line) throws IOException {
+ final List streams = read(line);
+ if (line.hasOption(OPTION_JSON)) {
+ final List> output = new ArrayList<>();
+ for (final BackupStream stream : streams) {
+ final Map item = new HashMap<>();
+ item.put("manifest", stream.getManifest());
+ item.put("recordCount", stream.getRecords().size());
+ item.put("eventGroupCount", stream.getEventGroups().size());
+ output.add(item);
+ }
+ System.out.println(GSON.toJson(output));
+ return 0;
+ }
+ for (final BackupStream stream : streams) {
+ System.out.println(
+ String.format(
+ CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
+ stream.getManifest().streamId,
+ stream.getRecords().size(),
+ stream.getEventGroups().size()));
+ }
+ return 0;
+ }
+
+ private static int verify(final CommandLine line) throws IOException {
+ final List streams = read(line);
+ long recordCount = 0;
+ long eventCount = 0;
+ for (final BackupStream stream : streams) {
+ recordCount += stream.getRecords().size();
+ eventCount += stream.getEventGroups().size();
+ }
+ System.out.println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B,
+ streams.size(),
+ recordCount,
+ eventCount));
+ return 0;
+ }
+
+ private static int export(final CommandLine line) throws IOException {
+ final Path source = Path.of(input(line)).toAbsolutePath().normalize();
+ final Path target =
+ Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
+ final List streams = read(line);
+ final Path root;
+ if (Files.isDirectory(source)) {
+ root = source;
+ } else if (Files.isRegularFile(source)
+ && "manifest.json".equals(source.getFileName().toString())) {
+ root = source.getParent();
+ } else {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
+ source));
+ }
+ if (target.startsWith(root)) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994);
+ }
+ if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
+ throw new java.nio.file.FileAlreadyExistsException(target.toString());
+ }
+ final Path parent = target.toAbsolutePath().normalize().getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp");
+ boolean published = false;
+ try {
+ try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) {
+ for (final Path file : exportFiles(root, streams)) {
+ final String entryName = root.relativize(file).toString().replace('\\', '/');
+ zip.putNextEntry(new ZipEntry(entryName));
+ Files.copy(file, zip);
+ zip.closeEntry();
+ }
+ }
+ try (final java.nio.channels.FileChannel archiveChannel =
+ java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) {
+ archiveChannel.force(true);
+ }
+ try {
+ Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE);
+ } catch (final java.nio.file.AtomicMoveNotSupportedException e) {
+ Files.move(temporary, target);
+ }
+ published = true;
+ } finally {
+ if (!published) {
+ Files.deleteIfExists(temporary);
+ }
+ }
+ System.out.println(
+ String.format(
+ CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target));
+ return 0;
+ }
+
+ private static List exportFiles(final Path root, final List streams)
+ throws IOException {
+ final Set files = new LinkedHashSet<>();
+ for (final BackupStream stream : streams) {
+ final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize();
+ if (!manifestPath.startsWith(root)) {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
+ root));
+ }
+ files.add(manifestPath);
+ final Path streamDirectory = manifestPath.getParent();
+ for (final LogicalBackupManifest.Segment segment : stream.getManifest().segments) {
+ files.add(streamDirectory.resolve(segment.file).normalize());
+ }
+ }
+ final List sortedFiles = new ArrayList<>(files);
+ sortedFiles.sort(Comparator.comparing(Path::toString));
+ return sortedFiles;
+ }
+
+ private static int importBackup(final CommandLine line) throws Exception {
+ final List streams = read(line);
+ final boolean dryRun = line.hasOption(OPTION_DRY_RUN);
+ if (dryRun) {
+ System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B);
+ for (final BackupStream stream : streams) {
+ System.out.println(
+ String.format(
+ CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
+ stream.getManifest().streamId,
+ stream.getRecords().size(),
+ stream.getEventGroups().size()));
+ }
+ return 0;
+ }
+ final String host = required(line, OPTION_HOST);
+ final int port = Integer.parseInt(required(line, OPTION_PORT));
+ final String user = line.getOptionValue(OPTION_USER, "root");
+ final String password = password(line);
+ final Path checkpoint = checkpointPath(line);
+ final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user);
+ try (final IoTDBSyncClient client =
+ new IoTDBSyncClient(
+ new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(),
+ host,
+ port,
+ false,
+ "",
+ "")) {
+ handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
+ long importedGroups = 0;
+ for (final BackupStream stream : streams) {
+ final String streamId = stream.getManifest().streamId;
+ final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L);
+ for (final EventGroup group : stream.getEventGroups()) {
+ if (group.getLastSequence() <= lastApplied) {
+ continue;
+ }
+ for (final LogicalBackupRecord record : group.getRequests()) {
+ transfer(client, record.toTPipeTransferReq());
+ }
+ checkpointState.appliedSequences.put(streamId, group.getLastSequence());
+ writeCheckpoint(checkpoint, checkpointState);
+ importedGroups++;
+ }
+ }
+ writeCheckpoint(checkpoint, checkpointState);
+ System.out.println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D,
+ importedGroups,
+ checkpoint));
+ }
+ return 0;
+ }
+
+ private static List read(final CommandLine line) throws IOException {
+ final String source = input(line);
+ try (final BackupInput input = BackupInput.open(Path.of(source))) {
+ return new LogicalBackupArchiveReader()
+ .read(input.getPath(), line.hasOption(OPTION_ALLOW_INCOMPLETE));
+ }
+ }
+
+ private static String input(final CommandLine line) throws IOException {
+ final String value = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
+ if (value == null || value.isBlank()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
+ }
+ return value;
+ }
+
+ private static String optionValue(
+ final CommandLine line, final String preferredOption, final String compatibilityOption) {
+ return line.hasOption(preferredOption)
+ ? line.getOptionValue(preferredOption)
+ : line.getOptionValue(compatibilityOption);
+ }
+
+ private static String required(final CommandLine line, final String option) throws IOException {
+ final String value = line.getOptionValue(option);
+ if (value == null || value.isBlank()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, option));
+ }
+ return value;
+ }
+
+ private static String password(final CommandLine line) throws IOException {
+ final boolean fromStandardInput = line.hasOption(OPTION_PASSWORD_STDIN);
+ final boolean fromEnvironment = line.hasOption(OPTION_PASSWORD_ENV);
+ if (fromStandardInput == fromEnvironment) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9);
+ }
+ if (fromStandardInput) {
+ final BufferedReader reader =
+ new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
+ final String value = reader.readLine();
+ if (value == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E);
+ }
+ return value;
+ }
+ if (fromEnvironment) {
+ final String value = System.getenv(line.getOptionValue(OPTION_PASSWORD_ENV));
+ if (value == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2);
+ }
+ return value;
+ }
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9);
+ }
+
+ private static void handshake(
+ final IoTDBSyncClient client,
+ final String timestampPrecision,
+ final String user,
+ final String password)
+ throws IOException, TException {
+ final Map params = new HashMap<>();
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId());
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false");
+ final TPipeTransferResp response =
+ client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params));
+ if (response == null
+ || response.getStatus() == null
+ || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697,
+ response == null ? "null" : response.getStatus()));
+ }
+ }
+
+ private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
+ throws TException, IOException {
+ final TPipeTransferResp response = client.pipeTransfer(request);
+ if (response == null
+ || response.getStatus() == null
+ || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && response.getStatus().getCode()
+ != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11,
+ request.getType(),
+ response == null ? "null" : response.getStatus()));
+ }
+ }
+
+ private static Path checkpointPath(final CommandLine line) throws IOException {
+ final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT);
+ if (configuredCheckpoint != null) {
+ return Path.of(configuredCheckpoint);
+ }
+ final Path source = Path.of(input(line)).toAbsolutePath().normalize();
+ return source.resolveSibling(source.getFileName() + ".import.checkpoint.json");
+ }
+
+ private static ImportCheckpoint readCheckpoint(
+ final Path checkpoint,
+ final List streams,
+ final String host,
+ final int port,
+ final String user)
+ throws IOException {
+ final Map sourceStreams = sourceStreamIdentities(streams);
+ if (!Files.exists(checkpoint)) {
+ final ImportCheckpoint state = new ImportCheckpoint();
+ state.formatName = CHECKPOINT_FORMAT_NAME;
+ state.formatVersion = CHECKPOINT_FORMAT_VERSION;
+ state.sourceStreams = sourceStreams;
+ state.targetHost = host;
+ state.targetPort = port;
+ state.targetUser = user;
+ return state;
+ }
+ final ImportCheckpoint state;
+ try {
+ state =
+ GSON.fromJson(
+ Files.readString(checkpoint, StandardCharsets.UTF_8), ImportCheckpoint.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint),
+ e);
+ }
+ if (state == null
+ || !CHECKPOINT_FORMAT_NAME.equals(state.formatName)
+ || !CHECKPOINT_FORMAT_VERSION.equals(state.formatVersion)
+ || state.sourceStreams == null
+ || state.appliedSequences == null) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint));
+ }
+ if (!sourceStreams.equals(state.sourceStreams)
+ || !host.equals(state.targetHost)
+ || port != state.targetPort
+ || !user.equals(state.targetUser)
+ || !checkpointSequencesAreValid(streams, state.appliedSequences)) {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D,
+ checkpoint));
+ }
+ return state;
+ }
+
+ static Map sourceStreamIdentities(final List streams) {
+ final Map identities = new LinkedHashMap<>();
+ for (final BackupStream stream : streams) {
+ final LogicalBackupManifest manifest = stream.getManifest();
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ updateDigest(digest, manifest.backupId);
+ updateDigest(digest, manifest.streamId);
+ updateDigest(digest, manifest.pipeName);
+ updateDigest(digest, manifest.pipeCreationTime);
+ updateDigest(digest, manifest.sourceClusterId);
+ updateDigest(digest, manifest.regionId);
+ updateDigest(digest, manifest.streamType);
+ updateDigest(digest, manifest.segments.size());
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ updateDigest(digest, segment.file);
+ updateDigest(digest, segment.segmentId);
+ updateDigest(digest, segment.firstSequence);
+ updateDigest(digest, segment.lastSequence);
+ updateDigest(digest, segment.recordCount);
+ updateDigest(digest, segment.sizeBytes);
+ updateDigest(digest, segment.sha256);
+ updateDigest(digest, segment.status);
+ }
+ updateDigest(digest, stream.getRecords().size());
+ for (final LogicalBackupRecord record : stream.getRecords()) {
+ digest.update(record.getRecordType().getCode());
+ digest.update(
+ ByteBuffer.allocate(Long.BYTES * 4 + Integer.BYTES + Byte.BYTES + Short.BYTES)
+ .putLong(record.getSequence())
+ .putLong(record.getEventGroupId().getMostSignificantBits())
+ .putLong(record.getEventGroupId().getLeastSignificantBits())
+ .putInt(record.getOperationIndex())
+ .putLong(record.getEventTime())
+ .put(record.getRequestVersion())
+ .putShort(record.getRequestType())
+ .array());
+ updateDigest(digest, record.getMetadata());
+ updateDigest(digest, record.getPayload());
+ }
+ identities.put(manifest.streamId, LogicalBackupFormat.toHex(digest.digest()));
+ }
+ return identities;
+ }
+
+ private static void updateDigest(final MessageDigest digest, final String value) {
+ if (value == null) {
+ digest.update((byte) 0);
+ return;
+ }
+ digest.update((byte) 1);
+ updateDigest(digest, value.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void updateDigest(final MessageDigest digest, final byte[] value) {
+ updateDigest(digest, value.length);
+ digest.update(value);
+ }
+
+ private static void updateDigest(final MessageDigest digest, final int value) {
+ digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array());
+ }
+
+ private static void updateDigest(final MessageDigest digest, final long value) {
+ digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array());
+ }
+
+ private static boolean checkpointSequencesAreValid(
+ final List streams, final Map appliedSequences) {
+ if (appliedSequences.size() > streams.size()) {
+ return false;
+ }
+ for (final BackupStream stream : streams) {
+ if (!appliedSequences.containsKey(stream.getManifest().streamId)) {
+ continue;
+ }
+ final Long applied = appliedSequences.get(stream.getManifest().streamId);
+ if (applied == null) {
+ return false;
+ }
+ if (applied == -1) {
+ continue;
+ }
+ if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) {
+ return false;
+ }
+ }
+ return appliedSequences.keySet().stream()
+ .allMatch(
+ streamId ->
+ streams.stream()
+ .anyMatch(stream -> stream.getManifest().streamId.equals(streamId)));
+ }
+
+ private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState)
+ throws IOException {
+ final Path parent = checkpoint.toAbsolutePath().normalize().getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp");
+ Files.deleteIfExists(temporary);
+ final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState));
+ try (final java.nio.channels.FileChannel checkpointChannel =
+ java.nio.channels.FileChannel.open(
+ temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ while (checkpointBytes.hasRemaining()) {
+ checkpointChannel.write(checkpointBytes);
+ }
+ checkpointChannel.force(true);
+ }
+ try {
+ Files.move(
+ temporary,
+ checkpoint,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final java.nio.file.AtomicMoveNotSupportedException e) {
+ Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ private static String getClusterId() {
+ final byte[] bytes = new byte[16];
+ new SecureRandom().nextBytes(bytes);
+ return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes);
+ }
+
+ private static void printUsage() {
+ final HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ new PrintWriter(System.out),
+ 120,
+ CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2,
+ CliMessages
+ .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E,
+ optionsForHelp(),
+ 2,
+ 2,
+ "",
+ true);
+ }
+
+ private static Options optionsForHelp() {
+ final Options options = new Options();
+ addOption(options, OPTION_INPUT, true, false);
+ addOption(options, OPTION_OUTPUT, true, false);
+ addOption(options, OPTION_RESUME, true, false);
+ addOption(options, OPTION_FORMAT, true, false);
+ addOption(options, OPTION_DEEP, false, false);
+ addOption(options, OPTION_HOST, true, false);
+ addOption(options, OPTION_PORT, true, false);
+ addOption(options, OPTION_USER, true, false);
+ addOption(options, OPTION_PASSWORD_STDIN, false, false);
+ addOption(options, OPTION_PASSWORD_ENV, true, false);
+ addOption(options, OPTION_CHECKPOINT, true, false);
+ addOption(options, OPTION_DRY_RUN, false, false);
+ addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
+ addOption(options, OPTION_JSON, false, false);
+ return options;
+ }
+
+ private static class ImportCheckpoint {
+ private String formatName;
+ private String formatVersion;
+ private Map sourceStreams = new LinkedHashMap<>();
+ private String targetHost;
+ private int targetPort;
+ private String targetUser;
+ private Map appliedSequences = new LinkedHashMap<>();
+ }
+
+ private static class BackupInput implements AutoCloseable {
+ private final Path path;
+ private final Path temporaryDirectory;
+
+ private BackupInput(final Path path, final Path temporaryDirectory) {
+ this.path = path;
+ this.temporaryDirectory = temporaryDirectory;
+ }
+
+ private static BackupInput open(final Path source) throws IOException {
+ final Path normalizedSource = source.toAbsolutePath().normalize();
+ if (!Files.isRegularFile(normalizedSource)
+ || "manifest.json".equals(normalizedSource.getFileName().toString())) {
+ return new BackupInput(normalizedSource, null);
+ }
+ final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-");
+ try {
+ extract(normalizedSource, temporaryDirectory);
+ return new BackupInput(temporaryDirectory, temporaryDirectory);
+ } catch (final IOException | RuntimeException e) {
+ deleteRecursively(temporaryDirectory);
+ throw e;
+ }
+ }
+
+ private static void extract(final Path archive, final Path target) throws IOException {
+ int entryCount = 0;
+ long totalBytes = 0;
+ final byte[] buffer = new byte[64 * 1024];
+ try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) {
+ ZipEntry entry;
+ while ((entry = zip.getNextEntry()) != null) {
+ if (++entryCount > MAX_ARCHIVE_ENTRIES) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ final Path output = target.resolve(entry.getName()).normalize();
+ if (!output.startsWith(target)) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152,
+ entry.getName()));
+ }
+ if (entry.isDirectory()) {
+ Files.createDirectories(output);
+ } else {
+ final Path parent = output.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ try (final OutputStream stream =
+ Files.newOutputStream(
+ output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ int length;
+ while ((length = zip.read(buffer)) >= 0) {
+ if (length == 0) {
+ continue;
+ }
+ totalBytes += length;
+ if (totalBytes > MAX_ARCHIVE_BYTES) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ stream.write(buffer, 0, length);
+ }
+ }
+ }
+ zip.closeEntry();
+ }
+ }
+ }
+
+ private Path getPath() {
+ return path;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (temporaryDirectory != null) {
+ deleteRecursively(temporaryDirectory);
+ }
+ }
+
+ private static void deleteRecursively(final Path directory) throws IOException {
+ if (!Files.exists(directory)) {
+ return;
+ }
+ try (final java.util.stream.Stream paths = Files.walk(directory)) {
+ for (final Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
+ Files.deleteIfExists(path);
+ }
+ }
+ }
+ }
+}
diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
new file mode 100644
index 000000000000..0fe45b22d897
--- /dev/null
+++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool.pipe;
+
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.commons.cli.ParseException;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
+
+public class PipeLogicalBackupToolTest {
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("source");
+ writeBackup(source);
+
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", source.toString(), "--deep"}));
+
+ final Path archive = root.resolve("backup.zip");
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "export",
+ "--input",
+ source.toString(),
+ "--output",
+ archive.toString(),
+ "--format=binary"
+ }));
+ Assert.assertTrue(Files.isRegularFile(archive));
+ try (final ZipFile zip = new ZipFile(archive.toFile())) {
+ Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal"));
+ Assert.assertNull(zip.getEntry(".backup.lock"));
+ Assert.assertEquals(2, zip.size());
+ }
+
+ Assert.assertEquals(
+ 0, PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()}));
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {"import", "--input", archive.toString(), "--dry-run"}));
+
+ final byte[] originalArchive = Files.readAllBytes(archive);
+ try {
+ PipeLogicalBackupTool.run(
+ new String[] {"export", "--input", source.toString(), "--output", archive.toString()});
+ Assert.fail();
+ } catch (final java.nio.file.FileAlreadyExistsException expected) {
+ Assert.assertArrayEquals(originalArchive, Files.readAllBytes(archive));
+ }
+ }
+
+ @Test
+ public void testCheckpointSourceIdentityIncludesContent() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ final Map before = PipeLogicalBackupTool.sourceStreamIdentities(streams);
+
+ streams.get(0).getManifest().segments.get(0).sha256 = "different-content";
+ final Map after = PipeLogicalBackupTool.sourceStreamIdentities(streams);
+
+ Assert.assertNotEquals(before, after);
+ }
+
+ @Test
+ public void testZipSlipIsRejected() throws Exception {
+ final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip");
+ try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive))) {
+ zip.putNextEntry(new ZipEntry("../outside"));
+ zip.write(1);
+ zip.closeEntry();
+ }
+
+ try {
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()});
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testPlaintextPasswordOptionIsRejected() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+
+ try {
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "import", "--source", source.toString(), "--dry-run", "--password", "secret"
+ });
+ Assert.fail();
+ } catch (final ParseException expected) {
+ // Expected.
+ }
+ }
+
+ private static void writeBackup(final Path directory) throws Exception {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "data-1";
+ manifest.streamType = "data";
+ manifest.timestampPrecision = "ms";
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) {
+ writer.writeEvent(
+ UUID.randomUUID(),
+ 1,
+ Collections.singletonList(
+ new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 10)
+ .setBody(ByteBuffer.wrap(new byte[] {3}))),
+ "metadata");
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index b818a52e8387..d530fc7cf1bd 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2643,4 +2643,15 @@ private DataNodePipeMessages() {}
"Interrupted while reading OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}";
public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 =
"Failed to read OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 =
+ "Unsupported logical backup event policy: %s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
+ "Unsupported logical backup resume policy: %s";
+ public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
+ "Unsupported Pipe event for logical backup: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 =
+ "Logical backup ID must resolve below the configured backup directory: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB =
+ "Logical backup directory must not be empty";
}
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index fe768a1f0181..49575e46abb9 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2467,4 +2467,15 @@ private DataNodePipeMessages() {}
"读取 OPC UA 服务器操作限制时被中断，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}";
public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 =
"读取 OPC UA 服务器操作限制失败，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 =
+ "不支持的逻辑备份事件策略：%s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
+ "不支持的逻辑备份恢复策略：%s";
+ public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
+ "逻辑备份不支持 Pipe 事件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 =
+ "Pipe 逻辑备份 ID 必须解析到配置的备份目录下：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB =
+ "Pipe 逻辑备份目录不能为空";
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
index c8e87890afd6..08061c954afc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
@@ -26,6 +26,7 @@
import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink;
import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink;
+import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink;
import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
@@ -74,6 +75,9 @@ protected void initConstructors() {
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WRITE_BACK_CONNECTOR.getPipePluginName(), WriteBackSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(),
+ () -> new LogicalBackupSink("data"));
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBDataRegionAsyncSink::new);
@@ -96,6 +100,9 @@ protected void initConstructors() {
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WRITE_BACK_SINK.getPipePluginName(), WriteBackSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(),
+ () -> new LogicalBackupSink("data"));
pluginConstructors.put(
BuiltinPipePlugin.SUBSCRIPTION_SINK.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
index 160ecf54c015..a4eba6400c17 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
@@ -23,6 +23,7 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink;
+import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -46,6 +47,9 @@ protected void initConstructors() {
IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(),
+ () -> new LogicalBackupSink("schema"));
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new);
@@ -59,6 +63,9 @@ protected void initConstructors() {
BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
+ pluginConstructors.put(
+ BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(),
+ () -> new LogicalBackupSink("schema"));
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
new file mode 100644
index 000000000000..5050107c05f9
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup;
+
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
+import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent;
+import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
+import org.apache.iotdb.pipe.api.PipeConnector;
+import org.apache.iotdb.pipe.api.annotation.TableModel;
+import org.apache.iotdb.pipe.api.annotation.TreeModel;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+
+@TreeModel
+@TableModel
+public class LogicalBackupSink implements PipeConnector {
+
+ private static final int FILE_PIECE_BYTES = 64 * 1024;
+ private LogicalBackupWriter writer;
+ private String pipeName;
+ private long creationTime;
+ private int regionId;
+ private boolean includeHeartbeat;
+ private String unsupportedEventPolicy;
+ private final String streamType;
+
+ public LogicalBackupSink() {
+ this("unknown");
+ }
+
+ public LogicalBackupSink(final String streamType) {
+ this.streamType = streamType;
+ }
+
+ @Override
+ public void validate(final PipeParameterValidator validator) throws Exception {
+ final PipeParameters parameters = validator.getParameters();
+ validator.validateSynonymAttributes(
+ Collections.singletonList(PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY),
+ Collections.singletonList(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY),
+ true);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY);
+ validateSynonymAttributes(
+ validator,
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
+ validator.validateAttributeValueRange(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ true,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
+ final String policy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+ if (!PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL.equals(policy)
+ && !PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(policy)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405,
+ policy));
+ }
+ }
+
+ private static void validateSynonymAttributes(
+ final PipeParameterValidator validator,
+ final String connectorAttribute,
+ final String sinkAttribute) {
+ validator.validateSynonymAttributes(
+ Collections.singletonList(connectorAttribute),
+ Collections.singletonList(sinkAttribute),
+ false);
+ }
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ pipeName = configuration.getRuntimeEnvironment().getPipeName();
+ creationTime = configuration.getRuntimeEnvironment().getCreationTime();
+ regionId = configuration.getRuntimeEnvironment().getRegionId();
+ final String root =
+ parameters.getStringByKeys(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY);
+ if (root == null || root.isBlank()) {
+ throw new PipeParameterNotValidException(
+ DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB);
+ }
+ final String backupId =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY),
+ pipeName + "-" + creationTime);
+ if (backupId == null || backupId.isBlank()) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId;
+ final Path backupRoot = Paths.get(root).toAbsolutePath().normalize();
+ final Path backupIdPath;
+ try {
+ backupIdPath = Path.of(backupId);
+ } catch (final RuntimeException e) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ if (backupIdPath.isAbsolute()
+ || backupIdPath.getNameCount() != 1
+ || ".".equals(backupId)
+ || "..".equals(backupId)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize();
+ if (!directory.startsWith(backupRoot)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages
+ .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
+ backupId));
+ }
+ final String resume =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+ final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume);
+ if (!append
+ && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)
+ && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9,
+ resume));
+ }
+ if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) {
+ directory =
+ directory.resolveSibling(
+ directory.getFileName()
+ + "-new-"
+ + System.currentTimeMillis()
+ + "-"
+ + UUID.randomUUID().toString().substring(0, 8));
+ }
+ final long segmentSize =
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE);
+ final int maxRecord =
+ parameters.getIntOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE);
+ final String fsync =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE)
+ .toUpperCase(Locale.ROOT);
+ final LogicalBackupWriter.FsyncPolicy fsyncPolicy =
+ LogicalBackupWriter.FsyncPolicy.valueOf(fsync);
+ final int fsyncBatch =
+ parameters.getIntOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE);
+ final long fsyncPeriod =
+ parameters.getLongOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE);
+ includeHeartbeat =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE);
+ unsupportedEventPolicy =
+ parameters
+ .getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE)
+ .toLowerCase(Locale.ROOT);
+
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = backupId;
+ manifest.pipeName = pipeName;
+ manifest.pipeCreationTime = creationTime;
+ manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId();
+ manifest.sourceVersion = IoTDBConstant.VERSION;
+ manifest.timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+ manifest.streamId = streamId;
+ manifest.regionId = regionId;
+ manifest.streamType = streamType;
+ manifest.sinkTaskId = streamId;
+ writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ segmentSize,
+ maxRecord,
+ fsyncPolicy,
+ fsyncBatch,
+ fsyncPeriod,
+ append);
+ }
+
+ @Override
+ public void handshake() {}
+
+ @Override
+ public void heartbeat() throws Exception {
+ if (writer != null) {
+ writer.heartbeat();
+ }
+ }
+
+ @Override
+ public void transfer(final TabletInsertionEvent event) throws Exception {
+ if (event instanceof PipeInsertNodeTabletInsertionEvent) {
+ final PipeInsertNodeTabletInsertionEvent insert = (PipeInsertNodeTabletInsertionEvent) event;
+ transferEnriched(insert, () -> requests(insert));
+ } else if (event instanceof PipeRawTabletInsertionEvent) {
+ final PipeRawTabletInsertionEvent raw = (PipeRawTabletInsertionEvent) event;
+ transferEnriched(
+ raw,
+ () ->
+ Collections.singletonList(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ raw.convertToTablet(),
+ raw.isAligned(),
+ raw.isTableModelEvent()
+ ? raw.getTableModelDatabaseName()
+ : raw.getTreeModelDatabaseName())));
+ } else {
+ handleUnsupported(event);
+ }
+ }
+
+ @Override
+ public void transfer(final TsFileInsertionEvent event) throws Exception {
+ if (!(event instanceof PipeTsFileInsertionEvent)) {
+ handleUnsupported(event);
+ return;
+ }
+ final PipeTsFileInsertionEvent tsFile = (PipeTsFileInsertionEvent) event;
+ final List requests = new ArrayList<>();
+ if (!tsFile.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ tsFile.consumeTabletInsertionEventsWithRetry(
+ tablet -> {
+ requests.add(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ tablet.convertToTablet(),
+ tablet.isAligned(),
+ tablet.isTableModelEvent()
+ ? tablet.getTableModelDatabaseName()
+ : tablet.getTreeModelDatabaseName()));
+ },
+ getClass().getName());
+ if (!requests.isEmpty()) {
+ writer.writeEvent(eventId(tsFile), System.currentTimeMillis(), requests, metadata(tsFile));
+ }
+ } finally {
+ event.close();
+ tsFile.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ @Override
+ public void transfer(final Event event) throws Exception {
+ if (event instanceof PipeDeleteDataNodeEvent) {
+ final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event;
+ transferEnriched(
+ deletion,
+ () ->
+ Collections.singletonList(
+ PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode())));
+ } else if (event instanceof PipeSchemaRegionWritePlanEvent) {
+ final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event;
+ transferEnriched(
+ schema,
+ () ->
+ Collections.singletonList(
+ PipeTransferPlanNodeReq.toTPipeTransferReq(schema.getPlanNode())));
+ } else if (event instanceof PipeSchemaRegionSnapshotEvent) {
+ transferSnapshot((PipeSchemaRegionSnapshotEvent) event);
+ } else if (event instanceof PipeHeartbeatEvent) {
+ if (includeHeartbeat) {
+ writer.writeControl(
+ LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event));
+ }
+ } else if (event instanceof PipeTerminateEvent) {
+ writer.writeControl(
+ LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event));
+ } else {
+ handleUnsupported(event);
+ }
+ }
+
+ private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws Exception {
+ if (!event.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ final List requests = new ArrayList<>();
+ final File[] files =
+ new File[] {
+ event.getMTreeSnapshotFile(),
+ event.getTagLogSnapshotFile(),
+ event.getAttributeSnapshotFile()
+ };
+ for (final File file : files) {
+ if (file == null) {
+ continue;
+ }
+ try (final FileInputStream input = new FileInputStream(file)) {
+ final byte[] piece = new byte[FILE_PIECE_BYTES];
+ long offset = 0;
+ int length;
+ while ((length = input.read(piece)) >= 0) {
+ if (length == 0) {
+ continue;
+ }
+ requests.add(
+ PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(
+ file.getName(), offset, Arrays.copyOf(piece, length)));
+ offset += length;
+ }
+ }
+ }
+ requests.add(
+ PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq(
+ event.getTreePattern().getPattern(),
+ event.getTablePattern().getDatabasePattern(),
+ event.getTablePattern().getTablePattern(),
+ event.getTreePattern().isTreeModelDataAllowedToBeCaptured(),
+ event.getTablePattern().isTableModelDataAllowedToBeCaptured(),
+ files[0] == null ? null : files[0].getName(),
+ files[0] == null ? 0 : files[0].length(),
+ files[1] == null ? null : files[1].getName(),
+ files[1] == null ? 0 : files[1].length(),
+ files[2] == null ? null : files[2].getName(),
+ files[2] == null ? 0 : files[2].length(),
+ event.getDatabaseName(),
+ event.toSealTypeString()));
+ writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event));
+ } finally {
+ event.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ private List requests(final PipeInsertNodeTabletInsertionEvent event)
+ throws IOException {
+ final String databaseName =
+ event.isTableModelEvent()
+ ? event.getTableModelDatabaseName()
+ : event.getTreeModelDatabaseName();
+ if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) {
+ return Collections.singletonList(
+ PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq(
+ event.getInsertNode(), databaseName));
+ }
+
+ final List tablets = event.convertToTablets();
+ final List requests = new ArrayList<>(tablets.size());
+ for (int index = 0; index < tablets.size(); index++) {
+ requests.add(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ tablets.get(index), event.isAligned(index), databaseName));
+ }
+ return requests;
+ }
+
+ private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier)
+ throws Exception {
+ if (!event.increaseReferenceCount(getClass().getName())) {
+ return;
+ }
+ try {
+ writer.writeEvent(
+ eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event));
+ } finally {
+ event.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+
+ @FunctionalInterface
+ private interface RequestSupplier {
+ List get() throws Exception;
+ }
+
+ private void handleUnsupported(final Event event) throws Exception {
+ if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) {
+ writer.recordSkippedEvent(
+ System.currentTimeMillis(), event == null ? "null" : event.getClass().getName());
+ return;
+ }
+ throw new PipeParameterNotValidException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97,
+ event == null ? "null" : event.getClass().getName()));
+ }
+
+ private UUID eventId(final EnrichedEvent event) {
+ return UUID.nameUUIDFromBytes(
+ (event.getClass().getName()
+ + '\0'
+ + event.getPipeName()
+ + '\0'
+ + event.getCreationTime()
+ + '\0'
+ + event.getRegionId()
+ + '\0'
+ + commitIds(event))
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ private String commitIds(final EnrichedEvent event) {
+ try {
+ return String.valueOf(event.getCommitIds());
+ } catch (final UnsupportedOperationException ignored) {
+ return Long.toString(event.getCommitId());
+ }
+ }
+
+ private String metadata(final Event event) {
+ if (event instanceof EnrichedEvent) {
+ final EnrichedEvent enriched = (EnrichedEvent) event;
+ return "pipe="
+ + enriched.getPipeName()
+ + ";creationTime="
+ + enriched.getCreationTime()
+ + ";regionId="
+ + enriched.getRegionId()
+ + ";commitIds="
+ + commitIds(enriched)
+ + ";eventType="
+ + event.getClass().getName();
+ }
+ return "eventType=" + (event == null ? "null" : event.getClass().getName());
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml
index 3bb68a29562c..a632dbff5dde 100644
--- a/iotdb-core/node-commons/pom.xml
+++ b/iotdb-core/node-commons/pom.xml
@@ -139,6 +139,10 @@
com.google.guava
guava
+
+ com.google.code.gson
+ gson
+
com.google.code.findbugs
jsr305
diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
new file mode 100644
index 000000000000..1629e2dd0002
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.i18n;
+
+public final class LogicalBackupMessages {
+
+ public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "directory";
+ public static final String EXCEPTION_MANIFEST_7F5CB74A = "manifest";
+ public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsyncPolicy";
+ public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "eventGroupId";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D =
+ "Logical backup directory already exists: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC =
+ "Logical backup event must contain at least one request";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 =
+ "Logical backup request body must not be null";
+ public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 =
+ "Logical backup record exceeds max-record-bytes";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E =
+ "Unsupported logical backup manifest format: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 =
+ "Logical backup manifest does not match %s: expected %s, found %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 =
+ "Logical backup directory is locked: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB =
+ "Logical backup writer is closed";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A =
+ "Incomplete logical backup segment header: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 =
+ "Invalid logical backup segment footer size: %s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 =
+ "Invalid logical backup record magic at offset %d in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 =
+ "Logical backup record header CRC mismatch at offset %d in %s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E =
+ "Invalid logical backup record length at offset %d in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 =
+ "Logical backup record payload CRC mismatch at offset %d in %s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D =
+ "Unknown logical backup record type %d in %s";
+ public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A =
+ "Nested logical backup event groups in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 =
+ "Logical backup commit without begin in %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F =
+ "Logical backup operation index mismatch at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B =
+ "Logical backup sequence mismatch at offset %d in %s";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 =
+ "Incomplete logical backup record tail: %s";
+ public static final String
+ EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 =
+ "Sealed logical backup segment contains an open event group: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 =
+ "Invalid logical backup segment magic: %s";
+ public static final String
+ EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB =
+ "Unsupported logical backup segment major version %d: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 =
+ "Logical backup segment header CRC mismatch: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 =
+ "Invalid logical backup segment footer magic: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D =
+ "Logical backup segment footer ID mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 =
+ "Logical backup segment digest mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 =
+ "Logical backup segment footer CRC mismatch: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB =
+ "Logical backup segment footer metadata mismatch: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 =
+ "Logical backup event ID %s was already written with a different digest";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 =
+ "Invalid logical backup writer configuration: segment-size, max-record, fsync-batch and"
+ + " fsync-period must be positive";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F =
+ "Logical backup request outside an event group at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B =
+ "Logical backup control record inside an event group at offset %d in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A =
+ "Logical backup segment ID mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 =
+ "Logical backup sequence gap in %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 =
+ "Logical backup has no segments";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 =
+ "Invalid logical backup segment path: %s";
+ public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E =
+ "No logical backup manifest found under %s";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE =
+ "Duplicate logical backup stream ID: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 =
+ "Logical backup manifest has no segments: %s";
+ public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B =
+ "Unlisted logical backup segment %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 =
+ "Logical backup segment metadata mismatch: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF =
+ "Logical backup segment is not sealed: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB =
+ "Logical backup manifest counters do not match segment contents: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E =
+ "Logical backup sequence is not continuous across segments: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 =
+ "Logical backup manifest is invalid: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 =
+ "Logical backup contains skipped events: %s";
+
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 =
+ "Logical backup request version %d is not supported";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 =
+ "Logical backup request type %d is not allowed";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 =
+ "Duplicate logical backup event group ID: %s";
+ public static final String
+ EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 =
+ "Symbolic links are not allowed in logical backup directory paths: %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD =
+ "Logical backup directory is unavailable: %s";
+
+ private LogicalBackupMessages() {}
+}
diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
new file mode 100644
index 000000000000..bfc2bed98fed
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.i18n;
+
+public final class LogicalBackupMessages {
+
+ public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "目录";
+ public static final String EXCEPTION_MANIFEST_7F5CB74A = "清单";
+ public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsync 策略";
+ public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "事件组 ID";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D =
+ "Pipe 逻辑备份目录已存在：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC =
+ "Pipe 逻辑备份事件必须至少包含一个请求";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 =
+ "Pipe 逻辑备份请求体不能为空";
+ public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 =
+ "Pipe 逻辑备份记录超过 max-record-bytes";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E =
+ "不支持的 Pipe 逻辑备份清单格式：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 =
+ "Pipe 逻辑备份清单的 %s 不匹配：预期 %s，实际 %s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 =
+ "Pipe 逻辑备份目录已被锁定：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB =
+ "Pipe 逻辑备份写入器已关闭";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A =
+ "Pipe 逻辑备份 segment 文件头不完整：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 =
+ "Pipe 逻辑备份 segment 文件尾大小无效：%s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 =
+ "偏移 %d 处的 Pipe 逻辑备份记录魔数无效，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 =
+ "偏移 %d 处的 Pipe 逻辑备份记录头 CRC 不匹配，文件：%s";
+ public static final String
+ EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E =
+ "偏移 %d 处的 Pipe 逻辑备份记录长度无效，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 =
+ "偏移 %d 处的 Pipe 逻辑备份记录载荷 CRC 不匹配，文件：%s";
+ public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D =
+ "未知的 Pipe 逻辑备份记录类型 %d，文件：%s";
+ public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A =
+ "Pipe 逻辑备份事件组发生嵌套，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 =
+ "Pipe 逻辑备份事件组缺少开始记录，文件：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F =
+ "偏移 %d 处的 Pipe 逻辑备份操作序号不匹配，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B =
+ "偏移 %d 处的 Pipe 逻辑备份 sequence 不连续，文件：%s";
+ public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 =
+ "Pipe 逻辑备份记录尾部不完整：%s";
+ public static final String
+ EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 =
+ "已封存的 Pipe 逻辑备份 segment 包含未提交事件组：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 =
+ "Pipe 逻辑备份 segment 魔数无效：%s";
+ public static final String
+ EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB =
+ "不支持的 Pipe 逻辑备份 segment 主版本 %d：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 =
+ "Pipe 逻辑备份 segment 文件头 CRC 不匹配：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 =
+ "Pipe 逻辑备份 segment 文件尾魔数无效：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D =
+ "Pipe 逻辑备份 segment 文件尾 ID 不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 =
+ "Pipe 逻辑备份 segment 摘要不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 =
+ "Pipe 逻辑备份 segment 文件尾 CRC 不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB =
+ "Pipe 逻辑备份 segment 文件尾元数据不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 =
+ "Pipe 逻辑备份事件 ID %s 已使用不同摘要写入";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 =
+ "Pipe 逻辑备份写入器配置无效：segment-size、max-record、fsync-batch 和 fsync-period"
+ + " 必须为正数";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F =
+ "偏移 %d 处的 Pipe 逻辑备份请求不在事件组中，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B =
+ "偏移 %d 处的 Pipe 逻辑备份控制记录位于事件组内，文件：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A =
+ "Pipe 逻辑备份 segment ID 不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 =
+ "Pipe 逻辑备份 sequence 在 %s 中不连续";
+ public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 =
+ "Pipe 逻辑备份不包含 segment";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 =
+ "Pipe 逻辑备份 segment 路径无效：%s";
+ public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E =
+ "在 %s 下未找到 Pipe 逻辑备份 manifest";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE =
+ "Pipe 逻辑备份 stream ID 重复：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 =
+ "Pipe 逻辑备份 manifest 不包含 segment：%s";
+ public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B =
+ "未在 manifest 中列出 Pipe 逻辑备份 segment：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 =
+ "Pipe 逻辑备份 segment 元数据不匹配：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF =
+ "Pipe 逻辑备份 segment 尚未封存：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB =
+ "Pipe 逻辑备份 manifest 计数与 segment 内容不匹配：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E =
+ "Pipe 逻辑备份 sequence 在 segment 之间不连续：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 =
+ "Pipe 逻辑备份 manifest 无效：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 =
+ "Pipe 逻辑备份包含已跳过的事件：%s";
+
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 =
+ "不支持逻辑备份请求版本 %d";
+ public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 =
+ "不允许逻辑备份请求类型 %d";
+ public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 =
+ "逻辑备份事件组 ID 重复：%s";
+ public static final String
+ EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 =
+ "逻辑备份目录路径中不允许使用符号链接：%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD =
+ "逻辑备份目录不可用：%s";
+
+ private LogicalBackupMessages() {}
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index 76b45d1e7989..2a10db843618 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -38,6 +38,7 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
+import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
@@ -95,6 +96,7 @@ public enum BuiltinPipePlugin {
OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class),
OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class),
WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class),
+ LOGICAL_BACKUP_CONNECTOR("logical-backup-connector", LogicalBackupSink.class),
DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class),
IOTDB_THRIFT_SINK("iotdb-thrift-sink", IoTDBThriftSink.class),
@@ -107,6 +109,7 @@ public enum BuiltinPipePlugin {
OPC_UA_SINK("opc-ua-sink", OpcUaSink.class),
OPC_DA_SINK("opc-da-sink", OpcDaSink.class),
WRITE_BACK_SINK("write-back-sink", WriteBackSink.class),
+ LOGICAL_BACKUP_SINK("logical-backup-sink", LogicalBackupSink.class),
SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class),
IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class),
// Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename.
@@ -178,6 +181,7 @@ public String getClassName() {
OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(),
OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(),
WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(),
+ LOGICAL_BACKUP_CONNECTOR.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
// Sinks
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java
new file mode 100644
index 000000000000..d5fb9b22021e
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.PlaceholderSink;
+
+public class LogicalBackupSink extends PlaceholderSink {}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
index f2d9ea283c82..6d5dac18c3d9 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java
@@ -80,6 +80,55 @@ public class PipeSinkConstant {
public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first";
public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true;
+ public static final String CONNECTOR_LOGICAL_BACKUP_DIR_KEY = "connector.dir";
+ public static final String SINK_LOGICAL_BACKUP_DIR_KEY = "sink.dir";
+ public static final String CONNECTOR_LOGICAL_BACKUP_ID_KEY = "connector.backup-id";
+ public static final String SINK_LOGICAL_BACKUP_ID_KEY = "sink.backup-id";
+ public static final String CONNECTOR_LOGICAL_BACKUP_RESUME_KEY = "connector.resume";
+ public static final String SINK_LOGICAL_BACKUP_RESUME_KEY = "sink.resume";
+ public static final String LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS = "fail-if-exists";
+ public static final String LOGICAL_BACKUP_RESUME_APPEND = "append";
+ public static final String LOGICAL_BACKUP_RESUME_NEW = "new";
+ public static final String LOGICAL_BACKUP_RESUME_DEFAULT_VALUE =
+ LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS;
+ public static final String CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY =
+ "connector.segment-size-bytes";
+ public static final String SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = "sink.segment-size-bytes";
+ public static final long LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE = 256L * MB;
+ public static final String CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY =
+ "connector.max-record-bytes";
+ public static final String SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = "sink.max-record-bytes";
+ public static final int LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE = 64 * 1024 * 1024;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "connector.fsync-policy";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "sink.fsync-policy";
+ public static final String LOGICAL_BACKUP_FSYNC_ALWAYS = "always";
+ public static final String LOGICAL_BACKUP_FSYNC_BATCH = "batch";
+ public static final String LOGICAL_BACKUP_FSYNC_PERIODIC = "periodic";
+ public static final String LOGICAL_BACKUP_FSYNC_NONE = "none";
+ public static final String LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE =
+ LOGICAL_BACKUP_FSYNC_ALWAYS;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY =
+ "connector.fsync-batch-operations";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY =
+ "sink.fsync-batch-operations";
+ public static final int LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE = 1000;
+ public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY =
+ "connector.fsync-period-ms";
+ public static final String SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = "sink.fsync-period-ms";
+ public static final long LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE = 1000;
+ public static final String CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY =
+ "connector.on-unsupported-event";
+ public static final String SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY =
+ "sink.on-unsupported-event";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL = "fail";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP = "skip";
+ public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE =
+ LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL;
+ public static final String CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY =
+ "connector.include-heartbeat";
+ public static final String SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = "sink.include-heartbeat";
+ public static final boolean LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE = false;
+
public static final String CONNECTOR_SERIALIZE_BY_REGION_KEY = "connector.serialize-by-region";
public static final String SINK_SERIALIZE_BY_REGION_KEY = "sink.serialize-by-region";
public static final boolean CONNECTOR_SERIALIZE_BY_REGION_DEFAULT_VALUE = false;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
new file mode 100644
index 000000000000..6ac9ca23f67b
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonParseException;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class LogicalBackupArchiveReader {
+
+ private static final Gson GSON = new Gson();
+ private static final Set REPLAYABLE_REQUEST_TYPES =
+ Set.of(
+ PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType(),
+ PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(),
+ PipeRequestType.TRANSFER_PLAN_NODE.getType(),
+ PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(),
+ PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType());
+
+ public List read(final Path source, final boolean allowIncomplete)
+ throws IOException {
+ final Path normalizedSource = source.toAbsolutePath().normalize();
+ final List manifests = discoverManifests(normalizedSource);
+ if (manifests.isEmpty()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E,
+ source));
+ }
+ final List streams = new ArrayList<>();
+ final Set streamIds = new HashSet<>();
+ LogicalBackupManifest archiveIdentity = null;
+ for (final Path manifest : manifests) {
+ final BackupStream stream = readStream(manifest, allowIncomplete);
+ if (archiveIdentity == null) {
+ archiveIdentity = stream.getManifest();
+ } else {
+ validateArchiveIdentity(archiveIdentity, stream.getManifest());
+ }
+ if (!streamIds.add(stream.getManifest().streamId)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE,
+ stream.getManifest().streamId));
+ }
+ streams.add(stream);
+ }
+ streams.sort(
+ Comparator.comparingInt(
+ (BackupStream stream) ->
+ "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1)
+ .thenComparing((BackupStream stream) -> stream.getManifest().streamId));
+ return Collections.unmodifiableList(streams);
+ }
+
+ private static void validateArchiveIdentity(
+ final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException {
+ validateArchiveIdentityField("backupId", expected.backupId, actual.backupId);
+ validateArchiveIdentityField("pipeName", expected.pipeName, actual.pipeName);
+ validateArchiveIdentityField(
+ "pipeCreationTime", expected.pipeCreationTime, actual.pipeCreationTime);
+ validateArchiveIdentityField(
+ "sourceClusterId", expected.sourceClusterId, actual.sourceClusterId);
+ validateArchiveIdentityField("sourceVersion", expected.sourceVersion, actual.sourceVersion);
+ validateArchiveIdentityField(
+ "timestampPrecision", expected.timestampPrecision, actual.timestampPrecision);
+ }
+
+ private static void validateArchiveIdentityField(
+ final String field, final Object expected, final Object actual) throws IOException {
+ if (!java.util.Objects.equals(expected, actual)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete)
+ throws IOException {
+ final LogicalBackupManifest manifest;
+ try {
+ manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath),
+ e);
+ }
+ validateManifest(manifest, manifestPath);
+ if (!allowIncomplete && manifest.skippedEventCount > 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
+ manifestPath));
+ }
+
+ final Path streamDirectory = manifestPath.getParent().toAbsolutePath().normalize();
+ final LogicalBackupSegmentReader segmentReader =
+ new LogicalBackupSegmentReader(manifest.maxRecordBytes);
+ final List records = new ArrayList<>();
+ final Map operationCounts = new LinkedHashMap<>();
+ final Set listedSegments = new HashSet<>();
+ long expectedSequence = -1;
+ long expectedSegmentId = 0;
+
+ for (int index = 0; index < manifest.segments.size(); index++) {
+ final LogicalBackupManifest.Segment segment = manifest.segments.get(index);
+ if (segment == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ "null"));
+ }
+ final Path segmentPath = resolveSegmentPath(streamDirectory, segment.file);
+ if (segmentPath == null
+ || Files.isSymbolicLink(segmentPath)
+ || !Files.isRegularFile(segmentPath, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ segment.file));
+ }
+ listedSegments.add(segmentPath);
+ final boolean lastSegment = index == manifest.segments.size() - 1;
+ final LogicalBackupSegmentReader.ScanResult scan =
+ segmentReader.scan(segmentPath, allowIncomplete && lastSegment);
+ if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF,
+ segmentPath));
+ }
+ if (scan.isSealed()) {
+ validateSegmentManifest(segment, scan, segmentPath);
+ }
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ validateReplayableRequest(record);
+ }
+
+ final List committedRecords =
+ scan.hasOpenEventGroup() ? removeOpenEventGroup(scan.getRecords()) : scan.getRecords();
+ for (final LogicalBackupRecord record : committedRecords) {
+ if (expectedSequence >= 0 && record.getSequence() != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E,
+ manifestPath));
+ }
+ expectedSequence = record.getSequence() + 1;
+ records.add(record);
+ operationCounts.merge(record.getRecordType().name(), 1L, Long::sum);
+ }
+ }
+
+ if (!allowIncomplete) {
+ validateUnlistedSegments(streamDirectory, listedSegments);
+ final long skippedEventCount =
+ operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L);
+ if (skippedEventCount > 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
+ manifestPath));
+ }
+ if (skippedEventCount != manifest.skippedEventCount
+ || !operationCounts.equals(manifest.operationCounts)
+ || (records.isEmpty()
+ ? manifest.firstSequence != -1 || manifest.lastSequence != -1
+ : records.get(0).getSequence() != manifest.firstSequence
+ || records.get(records.size() - 1).getSequence() != manifest.lastSequence)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB,
+ manifestPath));
+ }
+ }
+ return new BackupStream(manifestPath, manifest, records, toEventGroups(records));
+ }
+
+ private static void validateReplayableRequest(final LogicalBackupRecord record)
+ throws IOException {
+ if (record.getRecordType() != LogicalBackupRecordType.PIPE_REQUEST) {
+ return;
+ }
+ if (record.getRequestVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7,
+ record.getRequestVersion()));
+ }
+ if (!REPLAYABLE_REQUEST_TYPES.contains(record.getRequestType())) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38,
+ record.getRequestType()));
+ }
+ }
+
+ private static List discoverManifests(final Path source) throws IOException {
+ if (!Files.isSymbolicLink(source)
+ && Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS)
+ && LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())) {
+ return Collections.singletonList(source);
+ }
+ if (!Files.isDirectory(source)) {
+ return Collections.emptyList();
+ }
+ try (final Stream paths = Files.walk(source)) {
+ return paths
+ .filter(path -> !Files.isSymbolicLink(path))
+ .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS))
+ .filter(
+ path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString()))
+ .sorted()
+ .collect(Collectors.toList());
+ }
+ }
+
+ private static Path resolveSegmentPath(final Path streamDirectory, final String segmentFile) {
+ if (segmentFile == null) {
+ return null;
+ }
+ final Path relativePath;
+ try {
+ relativePath = Path.of(segmentFile);
+ } catch (final RuntimeException e) {
+ return null;
+ }
+ if (relativePath.isAbsolute() || relativePath.getNameCount() != 1) {
+ return null;
+ }
+ final Path segmentPath = streamDirectory.resolve(relativePath).normalize();
+ return segmentPath.startsWith(streamDirectory)
+ && segmentPath.getParent().equals(streamDirectory)
+ ? segmentPath
+ : null;
+ }
+
+ private static void validateManifest(
+ final LogicalBackupManifest manifest, final Path manifestPath) throws IOException {
+ if (manifest == null
+ || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName)
+ || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion)
+ || manifest.streamId == null
+ || manifest.backupId == null
+ || manifest.timestampPrecision == null
+ || manifest.maxRecordBytes <= 0
+ || manifest.maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES
+ || manifest.skippedEventCount < 0) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath));
+ }
+ if (manifest.segments == null || manifest.segments.isEmpty()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138,
+ manifestPath));
+ }
+ if (manifest.operationCounts == null) {
+ manifest.operationCounts = new LinkedHashMap<>();
+ }
+ }
+
+ private static void validateSegmentManifest(
+ final LogicalBackupManifest.Segment segment,
+ final LogicalBackupSegmentReader.ScanResult scan,
+ final Path segmentPath)
+ throws IOException {
+ final LogicalBackupSegmentReader.Footer footer = scan.getFooter();
+ if (!"SEALED".equals(segment.status)
+ || segment.firstSequence != footer.getFirstSequence()
+ || segment.lastSequence != footer.getLastSequence()
+ || segment.recordCount != footer.getRecordCount()
+ || segment.sizeBytes != scan.getValidLength()
+ || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3,
+ segmentPath));
+ }
+ }
+
+ private static void validateUnlistedSegments(
+ final Path streamDirectory, final Set listedSegments) throws IOException {
+ try (final Stream paths = Files.list(streamDirectory)) {
+ final Path unlisted =
+ paths
+ .filter(Files::isRegularFile)
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .map(path -> path.toAbsolutePath().normalize())
+ .filter(path -> !listedSegments.contains(path))
+ .findFirst()
+ .orElse(null);
+ if (unlisted != null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ unlisted));
+ }
+ }
+ }
+
+ private static List removeOpenEventGroup(
+ final List records) {
+ for (int i = records.size() - 1; i >= 0; i--) {
+ if (records.get(i).getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ return records.subList(0, i);
+ }
+ }
+ return records;
+ }
+
+ private static List toEventGroups(final List records)
+ throws IOException {
+ final List groups = new ArrayList<>();
+ final Set eventGroupIds = new HashSet<>();
+ UUID eventGroupId = null;
+ long firstSequence = -1;
+ String metadata = "";
+ final List requests = new ArrayList<>();
+ for (final LogicalBackupRecord record : records) {
+ if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ eventGroupId = record.getEventGroupId();
+ if (!eventGroupIds.add(eventGroupId)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89,
+ eventGroupId));
+ }
+ firstSequence = record.getSequence();
+ metadata = record.getMetadata();
+ requests.clear();
+ } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) {
+ requests.add(record);
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) {
+ groups.add(
+ new EventGroup(
+ eventGroupId,
+ firstSequence,
+ record.getSequence(),
+ metadata,
+ new ArrayList<>(requests)));
+ eventGroupId = null;
+ requests.clear();
+ }
+ }
+ return Collections.unmodifiableList(groups);
+ }
+
+ public static class BackupStream {
+ private final Path manifestPath;
+ private final LogicalBackupManifest manifest;
+ private final List records;
+ private final List eventGroups;
+
+ private BackupStream(
+ final Path manifestPath,
+ final LogicalBackupManifest manifest,
+ final List records,
+ final List eventGroups) {
+ this.manifestPath = manifestPath;
+ this.manifest = manifest;
+ this.records = Collections.unmodifiableList(records);
+ this.eventGroups = eventGroups;
+ }
+
+ public Path getManifestPath() {
+ return manifestPath;
+ }
+
+ public LogicalBackupManifest getManifest() {
+ return manifest;
+ }
+
+ public List getRecords() {
+ return records;
+ }
+
+ public List getEventGroups() {
+ return eventGroups;
+ }
+ }
+
+ public static class EventGroup {
+ private final UUID eventGroupId;
+ private final long firstSequence;
+ private final long lastSequence;
+ private final String metadata;
+ private final List requests;
+
+ private EventGroup(
+ final UUID eventGroupId,
+ final long firstSequence,
+ final long lastSequence,
+ final String metadata,
+ final List requests) {
+ this.eventGroupId = eventGroupId;
+ this.firstSequence = firstSequence;
+ this.lastSequence = lastSequence;
+ this.metadata = metadata;
+ this.requests = Collections.unmodifiableList(requests);
+ }
+
+ public UUID getEventGroupId() {
+ return eventGroupId;
+ }
+
+ public long getFirstSequence() {
+ return firstSequence;
+ }
+
+ public long getLastSequence() {
+ return lastSequence;
+ }
+
+ public String getMetadata() {
+ return metadata;
+ }
+
+ public List getRequests() {
+ return requests;
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java
new file mode 100644
index 000000000000..6309595202b5
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+import java.util.zip.CRC32C;
+
+public final class LogicalBackupFormat {
+
+ public static final String FORMAT_NAME = "iotdb-pipe-logical-backup";
+ public static final String FORMAT_VERSION = "1.0";
+ public static final short MAJOR_VERSION = 1;
+ public static final short MINOR_VERSION = 0;
+ public static final long SEGMENT_MAGIC = 0x494F545057414C31L;
+ public static final int RECORD_MAGIC = 0x50575231;
+ public static final int FOOTER_MAGIC = 0x50575346;
+ public static final int SEGMENT_HEADER_SIZE = 32;
+ public static final int RECORD_HEADER_SIZE = 56;
+ public static final int SEGMENT_FOOTER_SIZE = 96;
+ public static final int SHA256_SIZE = 32;
+ public static final int MAX_RECORD_BYTES = 256 * 1024 * 1024;
+ public static final String MANIFEST_FILE_NAME = "manifest.json";
+
+ private LogicalBackupFormat() {}
+
+ public static int crc32c(final byte[] bytes, final int offset, final int length) {
+ final CRC32C crc32c = new CRC32C();
+ crc32c.update(bytes, offset, length);
+ return (int) crc32c.getValue();
+ }
+
+ public static MessageDigest newSha256() {
+ try {
+ return MessageDigest.getInstance("SHA-256");
+ } catch (final NoSuchAlgorithmException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public static String toHex(final byte[] bytes) {
+ return HexFormat.of().formatHex(bytes);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
new file mode 100644
index 000000000000..3742ca88d801
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class LogicalBackupManifest {
+
+ public String formatName = LogicalBackupFormat.FORMAT_NAME;
+ public String formatVersion = LogicalBackupFormat.FORMAT_VERSION;
+ public String backupId;
+ public String status;
+ public String pipeName;
+ public long pipeCreationTime;
+ public String sourceClusterId;
+ public String sourceVersion;
+ public String timestampPrecision;
+ public String createdAt;
+ public String closedAt;
+ public String streamId;
+ public int regionId;
+ public String sinkTaskId;
+ public String streamType;
+ public String fsyncPolicy;
+ public long segmentSizeBytes;
+ public int maxRecordBytes;
+ public long firstSequence = -1;
+ public long lastSequence = -1;
+ public long lastDurableSequence = -1;
+ public String lastEventGroupId;
+ public String lastEventDigest;
+ public long lastEventFirstSequence = -1;
+ public long skippedEventCount;
+ public boolean recovered;
+ public Map operationCounts = new LinkedHashMap<>();
+ public List segments = new ArrayList<>();
+
+ public static class Segment {
+ public String file;
+ public long segmentId;
+ public long firstSequence = -1;
+ public long lastSequence = -1;
+ public long recordCount;
+ public long sizeBytes;
+ public String sha256;
+ public String status;
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java
new file mode 100644
index 000000000000..b779c4fc4aa3
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.UUID;
+
+public class LogicalBackupRecord {
+
+ private final LogicalBackupRecordType recordType;
+ private final long sequence;
+ private final UUID eventGroupId;
+ private final int operationIndex;
+ private final long eventTime;
+ private final byte requestVersion;
+ private final short requestType;
+ private final String metadata;
+ private final byte[] payload;
+
+ public LogicalBackupRecord(
+ final LogicalBackupRecordType recordType,
+ final long sequence,
+ final UUID eventGroupId,
+ final int operationIndex,
+ final long eventTime,
+ final byte requestVersion,
+ final short requestType,
+ final String metadata,
+ final byte[] payload) {
+ this.recordType = recordType;
+ this.sequence = sequence;
+ this.eventGroupId = eventGroupId;
+ this.operationIndex = operationIndex;
+ this.eventTime = eventTime;
+ this.requestVersion = requestVersion;
+ this.requestType = requestType;
+ this.metadata = metadata;
+ this.payload = payload;
+ }
+
+ public LogicalBackupRecordType getRecordType() {
+ return recordType;
+ }
+
+ public long getSequence() {
+ return sequence;
+ }
+
+ public UUID getEventGroupId() {
+ return eventGroupId;
+ }
+
+ public int getOperationIndex() {
+ return operationIndex;
+ }
+
+ public long getEventTime() {
+ return eventTime;
+ }
+
+ public byte getRequestVersion() {
+ return requestVersion;
+ }
+
+ public short getRequestType() {
+ return requestType;
+ }
+
+ public String getMetadata() {
+ return metadata;
+ }
+
+ public byte[] getPayload() {
+ return payload;
+ }
+
+ public TPipeTransferReq toTPipeTransferReq() {
+ return new TPipeTransferReq()
+ .setVersion(requestVersion)
+ .setType(requestType)
+ .setBody(ByteBuffer.wrap(payload));
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof LogicalBackupRecord)) {
+ return false;
+ }
+ final LogicalBackupRecord that = (LogicalBackupRecord) obj;
+ return sequence == that.sequence
+ && operationIndex == that.operationIndex
+ && eventTime == that.eventTime
+ && requestVersion == that.requestVersion
+ && requestType == that.requestType
+ && recordType == that.recordType
+ && eventGroupId.equals(that.eventGroupId)
+ && metadata.equals(that.metadata)
+ && Arrays.equals(payload, that.payload);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = eventGroupId.hashCode();
+ result = 31 * result + recordType.hashCode();
+ result = 31 * result + Long.hashCode(sequence);
+ result = 31 * result + operationIndex;
+ result = 31 * result + Long.hashCode(eventTime);
+ result = 31 * result + requestVersion;
+ result = 31 * result + requestType;
+ result = 31 * result + metadata.hashCode();
+ return 31 * result + Arrays.hashCode(payload);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java
new file mode 100644
index 000000000000..e188669548ca
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum LogicalBackupRecordType {
+ EVENT_BEGIN((byte) 1),
+ PIPE_REQUEST((byte) 2),
+ EVENT_COMMIT((byte) 3),
+ HEARTBEAT((byte) 4),
+ SKIPPED_EVENT((byte) 5),
+ STREAM_END((byte) 6);
+
+ private static final Map TYPE_MAP = new HashMap<>();
+
+ static {
+ for (final LogicalBackupRecordType type : values()) {
+ TYPE_MAP.put(type.code, type);
+ }
+ }
+
+ private final byte code;
+
+ LogicalBackupRecordType(final byte code) {
+ this.code = code;
+ }
+
+ public byte getCode() {
+ return code;
+ }
+
+ public static LogicalBackupRecordType valueOf(final byte code) {
+ return TYPE_MAP.get(code);
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java
new file mode 100644
index 000000000000..e400d6133c27
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupSegmentReader {
+
+ private final int maxRecordBytes;
+
+ public LogicalBackupSegmentReader(final int maxRecordBytes) {
+ this.maxRecordBytes = maxRecordBytes;
+ }
+
+ public ScanResult scan(final Path segment, final boolean allowIncompleteTail) throws IOException {
+ try (final RandomAccessFile reader = new RandomAccessFile(segment.toFile(), "r")) {
+ final long fileLength = reader.length();
+ if (fileLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw new EOFException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A,
+ segment));
+ }
+
+ final byte[] segmentHeader = readBytes(reader, LogicalBackupFormat.SEGMENT_HEADER_SIZE);
+ validateSegmentHeader(segmentHeader, segment);
+ final ByteBuffer segmentHeaderBuffer = wrap(segmentHeader);
+ segmentHeaderBuffer.position(Long.BYTES + Short.BYTES * 2);
+ final long segmentId = segmentHeaderBuffer.getLong();
+ final long createdAt = segmentHeaderBuffer.getLong();
+
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ digest.update(segmentHeader);
+ final List records = new ArrayList<>();
+ long validLength = LogicalBackupFormat.SEGMENT_HEADER_SIZE;
+ long lastCommittedLength = validLength;
+ boolean openEventGroup = false;
+ UUID openEventGroupId = null;
+ int nextOperationIndex = 0;
+ long expectedSequence = -1;
+ boolean incompleteTail = false;
+ Footer footer = null;
+
+ while (reader.getFilePointer() < fileLength) {
+ final long frameOffset = reader.getFilePointer();
+ if (fileLength - frameOffset < Integer.BYTES) {
+ incompleteTail = true;
+ break;
+ }
+
+ final int magic = reader.readInt();
+ reader.seek(frameOffset);
+ if (magic == LogicalBackupFormat.FOOTER_MAGIC) {
+ if (fileLength - frameOffset < LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ incompleteTail = true;
+ break;
+ }
+ if (fileLength - frameOffset > LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278,
+ segment));
+ }
+ final byte[] footerBytes = readBytes(reader, LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ footer =
+ parseAndValidateFooter(
+ footerBytes,
+ segmentId,
+ createdAt,
+ records.isEmpty() ? -1 : records.get(0).getSequence(),
+ records.isEmpty() ? -1 : records.get(records.size() - 1).getSequence(),
+ frameOffset,
+ records.size(),
+ digest.digest(),
+ segment);
+ validLength = reader.getFilePointer();
+ lastCommittedLength = validLength;
+ break;
+ }
+ if (magic != LogicalBackupFormat.RECORD_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5,
+ frameOffset,
+ segment));
+ }
+ if (fileLength - frameOffset < LogicalBackupFormat.RECORD_HEADER_SIZE) {
+ incompleteTail = true;
+ break;
+ }
+
+ final byte[] header = readBytes(reader, LogicalBackupFormat.RECORD_HEADER_SIZE);
+ final ByteBuffer headerBuffer = wrap(header);
+ headerBuffer.getInt();
+ final byte recordTypeCode = headerBuffer.get();
+ final long sequence = headerBuffer.getLong();
+ final UUID eventGroupId = new UUID(headerBuffer.getLong(), headerBuffer.getLong());
+ final int operationIndex = headerBuffer.getInt();
+ final long eventTime = headerBuffer.getLong();
+ final byte requestVersion = headerBuffer.get();
+ final short requestType = headerBuffer.getShort();
+ final int metadataLength = headerBuffer.getInt();
+ final int payloadLength = headerBuffer.getInt();
+ final int headerCrc = headerBuffer.getInt();
+ if (headerCrc
+ != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79,
+ frameOffset,
+ segment));
+ }
+ if (metadataLength < 0
+ || payloadLength < 0
+ || metadataLength > maxRecordBytes
+ || payloadLength > maxRecordBytes
+ || (long) metadataLength + payloadLength > maxRecordBytes) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E,
+ frameOffset,
+ segment));
+ }
+ final long remainingLength = (long) metadataLength + payloadLength + Integer.BYTES;
+ if (fileLength - reader.getFilePointer() < remainingLength) {
+ incompleteTail = true;
+ break;
+ }
+
+ final byte[] metadataBytes = readBytes(reader, metadataLength);
+ final byte[] payload = readBytes(reader, payloadLength);
+ final int frameCrc = reader.readInt();
+ final byte[] frameContent = new byte[metadataLength + payloadLength];
+ System.arraycopy(metadataBytes, 0, frameContent, 0, metadataLength);
+ System.arraycopy(payload, 0, frameContent, metadataLength, payloadLength);
+ if (frameCrc != LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552,
+ frameOffset,
+ segment));
+ }
+
+ final LogicalBackupRecordType recordType = LogicalBackupRecordType.valueOf(recordTypeCode);
+ if (recordType == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D,
+ recordTypeCode,
+ segment));
+ }
+ if (expectedSequence >= 0 && sequence != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B,
+ frameOffset,
+ segment));
+ }
+ final LogicalBackupRecord record =
+ new LogicalBackupRecord(
+ recordType,
+ sequence,
+ eventGroupId,
+ operationIndex,
+ eventTime,
+ requestVersion,
+ requestType,
+ new String(metadataBytes, StandardCharsets.UTF_8),
+ payload);
+ records.add(record);
+ digest.update(header);
+ digest.update(frameContent);
+ digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(frameCrc).array());
+ validLength = reader.getFilePointer();
+ if (recordType == LogicalBackupRecordType.EVENT_BEGIN) {
+ if (openEventGroup || operationIndex != -1) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A,
+ segment));
+ }
+ openEventGroup = true;
+ openEventGroupId = eventGroupId;
+ nextOperationIndex = 0;
+ } else if (recordType == LogicalBackupRecordType.EVENT_COMMIT) {
+ if (!openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541,
+ segment));
+ }
+ if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F,
+ frameOffset,
+ segment));
+ }
+ openEventGroup = false;
+ openEventGroupId = null;
+ lastCommittedLength = validLength;
+ } else if (recordType == LogicalBackupRecordType.PIPE_REQUEST) {
+ if (!openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F,
+ frameOffset,
+ segment));
+ }
+ if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F,
+ frameOffset,
+ segment));
+ }
+ nextOperationIndex++;
+ } else if (openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B,
+ frameOffset,
+ segment));
+ } else if (!openEventGroup) {
+ lastCommittedLength = validLength;
+ }
+ expectedSequence = sequence + 1;
+ }
+
+ if (incompleteTail && !allowIncompleteTail) {
+ throw new EOFException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7,
+ segment));
+ }
+ if (footer != null && openEventGroup) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8,
+ segment));
+ }
+ return new ScanResult(
+ segmentId,
+ createdAt,
+ records,
+ footer,
+ validLength,
+ lastCommittedLength,
+ incompleteTail,
+ openEventGroup);
+ }
+ }
+
+ private static void validateSegmentHeader(final byte[] header, final Path segment)
+ throws IOException {
+ final ByteBuffer buffer = wrap(header);
+ if (buffer.getLong() != LogicalBackupFormat.SEGMENT_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73,
+ segment));
+ }
+ final short majorVersion = buffer.getShort();
+ buffer.getShort();
+ if (majorVersion != LogicalBackupFormat.MAJOR_VERSION) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB,
+ majorVersion,
+ segment));
+ }
+ buffer.position(LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES);
+ final int headerCrc = buffer.getInt();
+ if (headerCrc
+ != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55,
+ segment));
+ }
+ }
+
+ private static Footer parseAndValidateFooter(
+ final byte[] bytes,
+ final long expectedSegmentId,
+ final long expectedCreatedAt,
+ final long expectedFirstSequence,
+ final long expectedLastSequence,
+ final long expectedValidBytes,
+ final long expectedRecordCount,
+ final byte[] expectedDigest,
+ final Path segment)
+ throws IOException {
+ final ByteBuffer buffer = wrap(bytes);
+ if (buffer.getInt() != LogicalBackupFormat.FOOTER_MAGIC) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4,
+ segment));
+ }
+ final long segmentId = buffer.getLong();
+ final long firstSequence = buffer.getLong();
+ final long lastSequence = buffer.getLong();
+ final long recordCount = buffer.getLong();
+ final long validBytes = buffer.getLong();
+ final long createdAt = buffer.getLong();
+ final long sealedAt = buffer.getLong();
+ final byte[] digest = new byte[LogicalBackupFormat.SHA256_SIZE];
+ buffer.get(digest);
+ final int footerCrc = buffer.getInt();
+ if (segmentId != expectedSegmentId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D,
+ segment));
+ }
+ if (createdAt != expectedCreatedAt
+ || firstSequence != expectedFirstSequence
+ || lastSequence != expectedLastSequence
+ || validBytes != expectedValidBytes
+ || recordCount != expectedRecordCount) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB,
+ segment));
+ }
+ if (!Arrays.equals(digest, expectedDigest)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116,
+ segment));
+ }
+ if (footerCrc
+ != LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91,
+ segment));
+ }
+ return new Footer(
+ segmentId,
+ firstSequence,
+ lastSequence,
+ recordCount,
+ validBytes,
+ createdAt,
+ sealedAt,
+ digest);
+ }
+
+ private static byte[] readBytes(final RandomAccessFile reader, final int length)
+ throws IOException {
+ final byte[] bytes = new byte[length];
+ reader.readFully(bytes);
+ return bytes;
+ }
+
+ private static ByteBuffer wrap(final byte[] bytes) {
+ return ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN);
+ }
+
+ public static class ScanResult {
+ private final long segmentId;
+ private final long createdAt;
+ private final List records;
+ private final Footer footer;
+ private final long validLength;
+ private final long lastCommittedLength;
+ private final boolean incompleteTail;
+ private final boolean openEventGroup;
+
+ private ScanResult(
+ final long segmentId,
+ final long createdAt,
+ final List records,
+ final Footer footer,
+ final long validLength,
+ final long lastCommittedLength,
+ final boolean incompleteTail,
+ final boolean openEventGroup) {
+ this.segmentId = segmentId;
+ this.createdAt = createdAt;
+ this.records = Collections.unmodifiableList(records);
+ this.footer = footer;
+ this.validLength = validLength;
+ this.lastCommittedLength = lastCommittedLength;
+ this.incompleteTail = incompleteTail;
+ this.openEventGroup = openEventGroup;
+ }
+
+ public long getSegmentId() {
+ return segmentId;
+ }
+
+ public long getCreatedAt() {
+ return createdAt;
+ }
+
+ public List getRecords() {
+ return records;
+ }
+
+ public Footer getFooter() {
+ return footer;
+ }
+
+ public long getValidLength() {
+ return validLength;
+ }
+
+ public long getLastCommittedLength() {
+ return lastCommittedLength;
+ }
+
+ public boolean hasIncompleteTail() {
+ return incompleteTail;
+ }
+
+ public boolean hasOpenEventGroup() {
+ return openEventGroup;
+ }
+
+ public boolean isSealed() {
+ return footer != null;
+ }
+ }
+
+ public static class Footer {
+ private final long segmentId;
+ private final long firstSequence;
+ private final long lastSequence;
+ private final long recordCount;
+ private final long validBytes;
+ private final long createdAt;
+ private final long sealedAt;
+ private final byte[] digest;
+
+ private Footer(
+ final long segmentId,
+ final long firstSequence,
+ final long lastSequence,
+ final long recordCount,
+ final long validBytes,
+ final long createdAt,
+ final long sealedAt,
+ final byte[] digest) {
+ this.segmentId = segmentId;
+ this.firstSequence = firstSequence;
+ this.lastSequence = lastSequence;
+ this.recordCount = recordCount;
+ this.validBytes = validBytes;
+ this.createdAt = createdAt;
+ this.sealedAt = sealedAt;
+ this.digest = digest;
+ }
+
+ public long getSegmentId() {
+ return segmentId;
+ }
+
+ public long getFirstSequence() {
+ return firstSequence;
+ }
+
+ public long getLastSequence() {
+ return lastSequence;
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public long getValidBytes() {
+ return validBytes;
+ }
+
+ public long getCreatedAt() {
+ return createdAt;
+ }
+
+ public long getSealedAt() {
+ return sealedAt;
+ }
+
+ public byte[] getDigest() {
+ return digest;
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
new file mode 100644
index 000000000000..9c8ba6f9a9a6
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
@@ -0,0 +1,1162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+
+public class LogicalBackupWriter implements Closeable {
+
+ public enum FsyncPolicy {
+ ALWAYS,
+ BATCH,
+ PERIODIC,
+ NONE
+ }
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+
+ private final Path directory;
+ private final Path manifestPath;
+ private final Path lockPath;
+ private final int maxRecordBytes;
+ private final long segmentSizeBytes;
+ private final FsyncPolicy fsyncPolicy;
+ private final int fsyncBatchOperations;
+ private final long fsyncPeriodMs;
+ private final LogicalBackupManifest manifest;
+ private final LogicalBackupSegmentReader reader;
+
+ private FileChannel channel;
+ private FileChannel lockChannel;
+ private FileLock directoryLock;
+ private long segmentId;
+ private long segmentCreatedAt;
+ private long segmentFirstSequence = -1;
+ private long segmentLastSequence = -1;
+ private long segmentRecordCount;
+ private MessageDigest segmentDigest;
+ private long nextSequence;
+ private int operationsSinceFsync;
+ private long lastFsyncAt;
+ private boolean closed;
+ private boolean streamEnded;
+ private boolean recoveredDuringOpen;
+
+ public LogicalBackupWriter(
+ final Path directory,
+ final LogicalBackupManifest manifest,
+ final long segmentSizeBytes,
+ final int maxRecordBytes,
+ final FsyncPolicy fsyncPolicy,
+ final int fsyncBatchOperations,
+ final long fsyncPeriodMs,
+ final boolean append)
+ throws IOException {
+ this.directory =
+ Objects.requireNonNull(directory, LogicalBackupMessages.EXCEPTION_DIRECTORY_5F8F22B8)
+ .toAbsolutePath()
+ .normalize();
+ this.manifest =
+ Objects.requireNonNull(manifest, LogicalBackupMessages.EXCEPTION_MANIFEST_7F5CB74A);
+ this.segmentSizeBytes = segmentSizeBytes;
+ this.maxRecordBytes = maxRecordBytes;
+ this.fsyncPolicy =
+ Objects.requireNonNull(fsyncPolicy, LogicalBackupMessages.EXCEPTION_FSYNC_POLICY_6D493614);
+ if (segmentSizeBytes
+ <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ || maxRecordBytes <= 0
+ || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES
+ || fsyncBatchOperations <= 0
+ || fsyncPeriodMs <= 0) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99);
+ }
+ this.fsyncBatchOperations = fsyncBatchOperations;
+ this.fsyncPeriodMs = fsyncPeriodMs;
+ this.manifestPath = this.directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ this.lockPath = this.directory.resolve(".backup.lock");
+ this.reader = new LogicalBackupSegmentReader(maxRecordBytes);
+ try {
+ prepareDirectory(append);
+ acquireLock();
+ if (Files.isSymbolicLink(manifestPath)) {
+ throw symbolicLinkException(manifestPath);
+ }
+ if (Files.exists(manifestPath)) {
+ if (!append) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D,
+ directory));
+ }
+ loadExistingManifest();
+ } else {
+ manifest.status = "WRITING";
+ manifest.createdAt = Instant.now().toString();
+ manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT);
+ manifest.segmentSizeBytes = segmentSizeBytes;
+ manifest.maxRecordBytes = maxRecordBytes;
+ writeManifest();
+ }
+ manifest.status = manifest.recovered ? "RECOVERED" : "WRITING";
+ manifest.closedAt = null;
+ nextSequence = Math.max(0, manifest.lastSequence + 1);
+ openOrCreateSegment();
+ rebuildManifestFromSegments();
+ if (recoveredDuringOpen) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ } else {
+ manifest.lastDurableSequence =
+ Math.max(-1, Math.min(manifest.lastDurableSequence, manifest.lastSequence));
+ }
+ updateActiveSegmentManifest();
+ manifest.status = manifest.recovered ? "RECOVERED" : "WRITING";
+ manifest.closedAt = null;
+ writeManifest();
+ lastFsyncAt = System.currentTimeMillis();
+ } catch (final IOException | RuntimeException e) {
+ if (channel != null) {
+ try {
+ channel.close();
+ } catch (final IOException closeException) {
+ e.addSuppressed(closeException);
+ }
+ channel = null;
+ }
+ releaseLock();
+ throw e;
+ }
+ }
+
+ private void prepareDirectory(final boolean append) throws IOException {
+ rejectSymbolicLinksInDirectoryPath();
+ if (append) {
+ Files.createDirectories(directory);
+ rejectSymbolicLinksInDirectoryPath();
+ return;
+ }
+ final Path parent = directory.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ rejectSymbolicLinksInDirectoryPath();
+ }
+ try {
+ Files.createDirectory(directory);
+ } catch (final FileAlreadyExistsException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D,
+ directory),
+ e);
+ }
+ rejectSymbolicLinksInDirectoryPath();
+ }
+
+ private void rejectSymbolicLinksInDirectoryPath() throws IOException {
+ Path current = directory.getRoot();
+ for (final Path part : directory) {
+ current = current == null ? part : current.resolve(part);
+ if (Files.isSymbolicLink(current)) {
+ throw symbolicLinkException(current);
+ }
+ }
+ }
+
+ private static IOException symbolicLinkException(final Path path) {
+ return new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569,
+ path));
+ }
+
+ public synchronized long writeEvent(
+ final UUID eventGroupId,
+ final long eventTime,
+ final List requests,
+ final String metadata)
+ throws IOException {
+ ensureOpen();
+ if (requests == null || requests.isEmpty()) {
+ throw new IOException(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC);
+ }
+ Objects.requireNonNull(eventGroupId, LogicalBackupMessages.EXCEPTION_EVENT_GROUP_ID_C6F6268A);
+ final String safeMetadata = metadata == null ? "" : metadata;
+ final String eventDigest = computeEventDigest(requests);
+ if (eventGroupId.toString().equals(manifest.lastEventGroupId)) {
+ if (eventDigest.equals(manifest.lastEventDigest)) {
+ updateActiveSegmentManifest();
+ writeManifest();
+ return manifest.lastEventFirstSequence;
+ }
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330,
+ eventGroupId));
+ }
+ rollBeforeEventIfNecessary(requests, safeMetadata);
+ final long eventStartPosition = channel.position();
+ final long firstSequence = nextSequence;
+ try {
+ appendRecord(
+ LogicalBackupRecordType.EVENT_BEGIN, eventGroupId, -1, eventTime, null, safeMetadata);
+ int operationIndex = 0;
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ appendRecord(
+ LogicalBackupRecordType.PIPE_REQUEST,
+ eventGroupId,
+ operationIndex++,
+ eventTime,
+ request,
+ safeMetadata);
+ }
+ appendRecord(
+ LogicalBackupRecordType.EVENT_COMMIT,
+ eventGroupId,
+ operationIndex,
+ eventTime,
+ null,
+ safeMetadata);
+ } catch (final IOException | RuntimeException e) {
+ try {
+ rollbackTo(eventStartPosition);
+ } catch (final IOException rollbackException) {
+ e.addSuppressed(rollbackException);
+ }
+ throw e;
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ }
+ updateActiveSegmentManifest();
+ manifest.lastEventGroupId = eventGroupId.toString();
+ manifest.lastEventDigest = eventDigest;
+ manifest.lastEventFirstSequence = firstSequence;
+ writeManifest();
+ return firstSequence;
+ }
+
+ public synchronized void writeControl(
+ final LogicalBackupRecordType recordType, final long eventTime, final String metadata)
+ throws IOException {
+ ensureOpen();
+ final String safeMetadata = metadata == null ? "" : metadata;
+ rollBeforeControlIfNecessary(safeMetadata);
+ final long recordStartPosition = channel.position();
+ try {
+ appendRecord(recordType, new UUID(0, 0), -1, eventTime, null, safeMetadata);
+ } catch (final IOException | RuntimeException e) {
+ try {
+ rollbackTo(recordStartPosition);
+ } catch (final IOException rollbackException) {
+ e.addSuppressed(rollbackException);
+ }
+ throw e;
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ }
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+
+ public synchronized void recordSkippedEvent(final long eventTime, final String metadata)
+ throws IOException {
+ writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata);
+ manifest.skippedEventCount++;
+ writeManifest();
+ }
+
+ public synchronized LogicalBackupManifest getManifest() {
+ return manifest;
+ }
+
+ public synchronized void heartbeat() throws IOException {
+ ensureOpen();
+ if (!Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)
+ || directoryLock == null
+ || !directoryLock.isValid()) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD,
+ directory));
+ }
+ if (forceIfNeeded()) {
+ manifest.lastDurableSequence = manifest.lastSequence;
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+ }
+
+ private void appendRecord(
+ final LogicalBackupRecordType recordType,
+ final UUID eventGroupId,
+ final int operationIndex,
+ final long eventTime,
+ final TPipeTransferReq request,
+ final String metadata)
+ throws IOException {
+ final byte[] metadataBytes = metadata.getBytes(StandardCharsets.UTF_8);
+ final byte[] payload =
+ request == null
+ ? new byte[0]
+ : java.util.Arrays.copyOf(request.getBody(), request.getBody().length);
+ if ((long) metadataBytes.length + payload.length > maxRecordBytes) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996);
+ }
+ final long requiredBytes =
+ LogicalBackupFormat.RECORD_HEADER_SIZE
+ + metadataBytes.length
+ + payload.length
+ + Integer.BYTES;
+ final long sequence = nextSequence++;
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.RECORD_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putInt(LogicalBackupFormat.RECORD_MAGIC);
+ header.put(recordType.getCode());
+ header.putLong(sequence);
+ header.putLong(eventGroupId.getMostSignificantBits());
+ header.putLong(eventGroupId.getLeastSignificantBits());
+ header.putInt(operationIndex);
+ header.putLong(eventTime);
+ header.put(request == null ? (byte) 0 : request.getVersion());
+ header.putShort(request == null ? (short) 0 : request.getType());
+ header.putInt(metadataBytes.length);
+ header.putInt(payload.length);
+ header.putInt(0);
+ final byte[] headerBytes = header.array();
+ final int headerCrc =
+ LogicalBackupFormat.crc32c(headerBytes, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4);
+ ByteBuffer.wrap(headerBytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(LogicalBackupFormat.RECORD_HEADER_SIZE - 4, headerCrc);
+ final byte[] frameContent = new byte[metadataBytes.length + payload.length];
+ System.arraycopy(metadataBytes, 0, frameContent, 0, metadataBytes.length);
+ System.arraycopy(payload, 0, frameContent, metadataBytes.length, payload.length);
+ final int frameCrc = LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length);
+
+ writeFully(ByteBuffer.wrap(headerBytes));
+ writeFully(ByteBuffer.wrap(frameContent));
+ writeFully(
+ ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).flip());
+ segmentDigest.update(headerBytes);
+ segmentDigest.update(frameContent);
+ segmentDigest.update(
+ ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).array());
+ if (segmentFirstSequence < 0) {
+ segmentFirstSequence = sequence;
+ }
+ segmentLastSequence = sequence;
+ segmentRecordCount++;
+ manifest.firstSequence = manifest.firstSequence < 0 ? sequence : manifest.firstSequence;
+ manifest.lastSequence = sequence;
+ final String countKey = recordType.name();
+ manifest.operationCounts.put(countKey, manifest.operationCounts.getOrDefault(countKey, 0L) + 1);
+ operationsSinceFsync++;
+ streamEnded = recordType == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private void openOrCreateSegment() throws IOException {
+ if (manifest.segments.isEmpty()) {
+ segmentId = 0;
+ validateSegmentFiles(segmentFileName(segmentId));
+ openNewSegment();
+ return;
+ }
+ final LogicalBackupManifest.Segment last = manifest.segments.get(manifest.segments.size() - 1);
+ final Path segmentPath = resolveExistingSegmentPath(last.file);
+ LogicalBackupSegmentReader.ScanResult scan = scanLastSegmentForAppend(segmentPath, last);
+ if (scan.isSealed()) {
+ if (!"SEALED".equals(last.status)) {
+ manifest.recovered = true;
+ recoveredDuringOpen = true;
+ }
+ segmentId = scan.getSegmentId() + 1;
+ // Validate every existing sealed segment before creating a new file in the backup.
+ rebuildManifestFromSegments();
+ validateSegmentFiles(segmentFileName(segmentId));
+ openNewSegment();
+ return;
+ }
+ validateSegmentFiles(null);
+ if (scan.hasIncompleteTail() || scan.hasOpenEventGroup()) {
+ try (final FileChannel truncateChannel =
+ FileChannel.open(segmentPath, StandardOpenOption.WRITE)) {
+ truncateChannel.truncate(scan.getLastCommittedLength());
+ truncateChannel.force(true);
+ }
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ recoveredDuringOpen = true;
+ scan = reader.scan(segmentPath, false);
+ }
+ if (!scan.getRecords().isEmpty()) {
+ nextSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence() + 1;
+ manifest.lastSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence();
+ }
+ segmentId = scan.getSegmentId();
+ openExistingSegment(segmentPath, scan);
+ }
+
+ private LogicalBackupSegmentReader.ScanResult scanLastSegmentForAppend(
+ final Path segmentPath, final LogicalBackupManifest.Segment segment) throws IOException {
+ try {
+ return reader.scan(segmentPath, true);
+ } catch (final IOException scanException) {
+ final long recoverableLength =
+ "SEALED".equals(segment.status)
+ ? -1
+ : findRecoverableTailStart(segmentPath, segment.sizeBytes);
+ if (recoverableLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw scanException;
+ }
+ try (final FileChannel truncateChannel =
+ FileChannel.open(segmentPath, StandardOpenOption.WRITE)) {
+ truncateChannel.truncate(recoverableLength);
+ truncateChannel.force(true);
+ }
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ recoveredDuringOpen = true;
+ return reader.scan(segmentPath, false);
+ }
+ }
+
+ private static long findRecoverableTailStart(
+ final Path segmentPath, final long manifestSegmentSize) throws IOException {
+ final long fileSize = Files.size(segmentPath);
+ if (fileSize
+ >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ final ByteBuffer magic = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN);
+ try (final FileChannel readChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) {
+ readChannel.position(fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ while (magic.hasRemaining()) {
+ if (readChannel.read(magic) < 0) {
+ break;
+ }
+ }
+ }
+ if (!magic.hasRemaining() && magic.flip().getInt() == LogicalBackupFormat.FOOTER_MAGIC) {
+ return fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ }
+ }
+ if (manifestSegmentSize >= LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && manifestSegmentSize < fileSize
+ && fileSize - manifestSegmentSize <= LogicalBackupFormat.SEGMENT_FOOTER_SIZE) {
+ return manifestSegmentSize;
+ }
+ return -1;
+ }
+
+ private void openExistingSegment(
+ final Path segmentPath, final LogicalBackupSegmentReader.ScanResult scan) throws IOException {
+ channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ);
+ channel.position(channel.size());
+ segmentCreatedAt = scan.getCreatedAt();
+ segmentDigest = LogicalBackupFormat.newSha256();
+ try (final FileChannel digestChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) {
+ final ByteBuffer digestBuffer = ByteBuffer.allocate(64 * 1024);
+ long remaining = scan.getValidLength();
+ while (remaining > 0) {
+ digestBuffer.clear();
+ digestBuffer.limit((int) Math.min(digestBuffer.capacity(), remaining));
+ final int read = digestChannel.read(digestBuffer);
+ if (read < 0) {
+ break;
+ }
+ remaining -= read;
+ segmentDigest.update(digestBuffer.array(), 0, read);
+ }
+ }
+ segmentFirstSequence = -1;
+ segmentLastSequence = -1;
+ segmentRecordCount = 0;
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ segmentFirstSequence = segmentFirstSequence < 0 ? record.getSequence() : segmentFirstSequence;
+ segmentLastSequence = record.getSequence();
+ segmentRecordCount++;
+ }
+ streamEnded =
+ !scan.getRecords().isEmpty()
+ && scan.getRecords().get(scan.getRecords().size() - 1).getRecordType()
+ == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private void openNewSegment() throws IOException {
+ final String fileName = segmentFileName(segmentId);
+ final Path segmentPath = resolveSegmentPath(fileName);
+ final Path temporarySegmentPath = resolveSegmentPath(fileName + ".tmp");
+ if (Files.exists(segmentPath, LinkOption.NOFOLLOW_LINKS)) {
+ adoptOrphanSegment(resolveExistingSegmentPath(fileName), fileName);
+ return;
+ }
+
+ // A temporary segment is never exposed through the manifest and contains only a header, so it
+ // is safe to discard after an interrupted creation attempt.
+ Files.deleteIfExists(temporarySegmentPath);
+ segmentCreatedAt = System.currentTimeMillis();
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putLong(LogicalBackupFormat.SEGMENT_MAGIC);
+ header.putShort(LogicalBackupFormat.MAJOR_VERSION);
+ header.putShort(LogicalBackupFormat.MINOR_VERSION);
+ header.putLong(segmentId);
+ header.putLong(segmentCreatedAt);
+ header.putInt(0);
+ final byte[] bytes = header.array();
+ ByteBuffer.wrap(bytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4,
+ LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4));
+ try (final FileChannel temporaryChannel =
+ FileChannel.open(
+ temporarySegmentPath,
+ StandardOpenOption.CREATE_NEW,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.READ)) {
+ final ByteBuffer headerBuffer = ByteBuffer.wrap(bytes);
+ while (headerBuffer.hasRemaining()) {
+ temporaryChannel.write(headerBuffer);
+ }
+ temporaryChannel.force(true);
+ }
+ try {
+ Files.move(temporarySegmentPath, segmentPath, StandardCopyOption.ATOMIC_MOVE);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(temporarySegmentPath, segmentPath);
+ }
+
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ openExistingSegment(segmentPath, scan);
+ addActiveSegmentToManifest(fileName);
+ }
+
+ private void adoptOrphanSegment(final Path segmentPath, final String fileName)
+ throws IOException {
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ if (scan.getSegmentId() != segmentId
+ || scan.isSealed()
+ || !scan.getRecords().isEmpty()
+ || scan.getValidLength() != LogicalBackupFormat.SEGMENT_HEADER_SIZE) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ segmentPath));
+ }
+ openExistingSegment(segmentPath, scan);
+ addActiveSegmentToManifest(fileName);
+ }
+
+ private void addActiveSegmentToManifest(final String fileName) throws IOException {
+ final LogicalBackupManifest.Segment segment = new LogicalBackupManifest.Segment();
+ segment.file = fileName;
+ segment.segmentId = segmentId;
+ segment.status = "ACTIVE";
+ manifest.segments.add(segment);
+ writeManifest();
+ }
+
+ private void validateSegmentFiles(final String allowedOrphanFile) throws IOException {
+ final Set listedFiles = new HashSet<>();
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ listedFiles.add(segment.file);
+ }
+ try (final DirectoryStream segmentFiles = Files.newDirectoryStream(directory, "*.pwal")) {
+ for (final Path segmentFile : segmentFiles) {
+ final String fileName = segmentFile.getFileName().toString();
+ if (!listedFiles.contains(fileName) && !fileName.equals(allowedOrphanFile)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B,
+ segmentFile));
+ }
+ }
+ }
+ }
+
+ private static String segmentFileName(final long id) {
+ return String.format(Locale.ROOT, "segment-%020d.pwal", id);
+ }
+
+ private void sealSegment() throws IOException {
+ if (channel == null || segmentRecordCount == 0) {
+ return;
+ }
+ final byte[] digest = segmentDigest.digest();
+ final ByteBuffer footer =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ footer.putInt(LogicalBackupFormat.FOOTER_MAGIC);
+ footer.putLong(segmentId);
+ footer.putLong(segmentFirstSequence);
+ footer.putLong(segmentLastSequence);
+ footer.putLong(segmentRecordCount);
+ footer.putLong(channel.position());
+ footer.putLong(segmentCreatedAt);
+ footer.putLong(System.currentTimeMillis());
+ footer.put(digest);
+ footer.putInt(0);
+ final byte[] footerBytes = footer.array();
+ ByteBuffer.wrap(footerBytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4,
+ LogicalBackupFormat.crc32c(
+ footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4));
+ writeFully(ByteBuffer.wrap(footerBytes));
+ channel.force(true);
+ final LogicalBackupManifest.Segment segment =
+ manifest.segments.get(manifest.segments.size() - 1);
+ segment.firstSequence = segmentFirstSequence;
+ segment.lastSequence = segmentLastSequence;
+ segment.recordCount = segmentRecordCount;
+ segment.sizeBytes = channel.position();
+ segment.sha256 = LogicalBackupFormat.toHex(digest);
+ segment.status = "SEALED";
+ manifest.lastDurableSequence = segmentLastSequence;
+ channel.close();
+ channel = null;
+ writeManifest();
+ }
+
+ private boolean forceIfNeeded() throws IOException {
+ if (channel == null) {
+ return false;
+ }
+ final long now = System.currentTimeMillis();
+ final boolean shouldForce =
+ fsyncPolicy == FsyncPolicy.ALWAYS
+ || (fsyncPolicy == FsyncPolicy.BATCH && operationsSinceFsync >= fsyncBatchOperations)
+ || (fsyncPolicy == FsyncPolicy.PERIODIC && now - lastFsyncAt >= fsyncPeriodMs);
+ if (shouldForce) {
+ channel.force(true);
+ operationsSinceFsync = 0;
+ lastFsyncAt = now;
+ }
+ return shouldForce;
+ }
+
+ private void writeManifest() throws IOException {
+ final Path temporary =
+ manifestPath.resolveSibling(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp");
+ final byte[] manifestBytes = GSON.toJson(manifest).getBytes(StandardCharsets.UTF_8);
+ Files.deleteIfExists(temporary);
+ try (final FileChannel manifestChannel =
+ FileChannel.open(temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ final ByteBuffer manifestBuffer = ByteBuffer.wrap(manifestBytes);
+ while (manifestBuffer.hasRemaining()) {
+ manifestChannel.write(manifestBuffer);
+ }
+ manifestChannel.force(true);
+ }
+ try {
+ Files.move(
+ temporary,
+ manifestPath,
+ StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(temporary, manifestPath, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ private void loadExistingManifest() throws IOException {
+ final LogicalBackupManifest existing;
+ try {
+ existing =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ } catch (final JsonParseException | NullPointerException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath),
+ e);
+ }
+ if (existing == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7,
+ manifestPath));
+ }
+ if (!LogicalBackupFormat.FORMAT_NAME.equals(existing.formatName)
+ || !LogicalBackupFormat.FORMAT_VERSION.equals(existing.formatVersion)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E,
+ manifestPath));
+ }
+ validateManifestIdentity("backupId", manifest.backupId, existing.backupId);
+ validateManifestIdentity("streamId", manifest.streamId, existing.streamId);
+ validateManifestIdentity("pipeName", manifest.pipeName, existing.pipeName);
+ validateManifestIdentity("streamType", manifest.streamType, existing.streamType);
+ validateManifestIdentity("sourceClusterId", manifest.sourceClusterId, existing.sourceClusterId);
+ validateManifestIdentity("sourceVersion", manifest.sourceVersion, existing.sourceVersion);
+ validateManifestIdentity(
+ "timestampPrecision", manifest.timestampPrecision, existing.timestampPrecision);
+ validateManifestIdentity("sinkTaskId", manifest.sinkTaskId, existing.sinkTaskId);
+ validateManifestIdentity(
+ "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy);
+ validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes);
+ validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes);
+ if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ "pipeCreationTime",
+ manifest.pipeCreationTime,
+ existing.pipeCreationTime));
+ }
+ if (manifest.regionId != existing.regionId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ "regionId",
+ manifest.regionId,
+ existing.regionId));
+ }
+ manifest.status = existing.status;
+ manifest.backupId = existing.backupId;
+ manifest.pipeName = existing.pipeName;
+ manifest.pipeCreationTime = existing.pipeCreationTime;
+ manifest.sourceClusterId = existing.sourceClusterId;
+ manifest.sourceVersion = existing.sourceVersion;
+ manifest.timestampPrecision = existing.timestampPrecision;
+ manifest.streamId = existing.streamId;
+ manifest.regionId = existing.regionId;
+ manifest.sinkTaskId = existing.sinkTaskId;
+ manifest.streamType = existing.streamType;
+ manifest.firstSequence = existing.firstSequence;
+ manifest.lastSequence = existing.lastSequence;
+ manifest.lastDurableSequence = existing.lastDurableSequence;
+ manifest.recovered = existing.recovered;
+ manifest.segments = existing.segments == null ? new ArrayList<>() : existing.segments;
+ manifest.operationCounts =
+ existing.operationCounts == null
+ ? new java.util.LinkedHashMap<>()
+ : existing.operationCounts;
+ manifest.createdAt = existing.createdAt;
+ manifest.closedAt = existing.closedAt;
+ manifest.fsyncPolicy = existing.fsyncPolicy;
+ manifest.segmentSizeBytes = existing.segmentSizeBytes;
+ manifest.maxRecordBytes = existing.maxRecordBytes;
+ manifest.lastEventGroupId = existing.lastEventGroupId;
+ manifest.lastEventDigest = existing.lastEventDigest;
+ manifest.lastEventFirstSequence = existing.lastEventFirstSequence;
+ manifest.skippedEventCount = existing.skippedEventCount;
+ }
+
+ private void validateManifestIdentity(
+ final String field, final String expected, final String actual) throws IOException {
+ if (expected != null && !Objects.equals(expected, actual)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private void validateManifestConfiguration(
+ final String field, final long expected, final long actual) throws IOException {
+ if (expected != actual) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1,
+ field,
+ expected,
+ actual));
+ }
+ }
+
+ private void acquireLock() throws IOException {
+ try {
+ if (Files.isSymbolicLink(lockPath)) {
+ throw symbolicLinkException(lockPath);
+ }
+ lockChannel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ directoryLock = lockChannel.tryLock();
+ if (directoryLock == null) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800,
+ directory));
+ }
+ lockChannel.truncate(0);
+ lockChannel.write(
+ StandardCharsets.UTF_8.encode(Long.toString(ProcessHandle.current().pid())));
+ lockChannel.force(true);
+ } catch (final OverlappingFileLockException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800,
+ directory),
+ e);
+ }
+ }
+
+ private void releaseLock() {
+ try {
+ if (directoryLock != null) {
+ directoryLock.release();
+ directoryLock = null;
+ }
+ } catch (final IOException ignored) {
+ // Best effort during failure/close.
+ }
+ try {
+ if (lockChannel != null) {
+ lockChannel.close();
+ lockChannel = null;
+ }
+ } catch (final IOException ignored) {
+ // Best effort during failure/close.
+ }
+ }
+
+ private void writeFully(final ByteBuffer buffer) throws IOException {
+ while (buffer.hasRemaining()) {
+ channel.write(buffer);
+ }
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed || channel == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB);
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ try {
+ if (channel != null) {
+ if (!streamEnded) {
+ writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), "");
+ }
+ sealSegment();
+ }
+ manifest.status = manifest.recovered ? "RECOVERED" : "SEALED";
+ manifest.closedAt = Instant.now().toString();
+ manifest.lastDurableSequence = manifest.lastSequence;
+ writeManifest();
+ } finally {
+ if (channel != null) {
+ channel.close();
+ channel = null;
+ }
+ closed = true;
+ releaseLock();
+ }
+ }
+
+ private void rollBeforeEventIfNecessary(
+ final List requests, final String metadata) throws IOException {
+ final long metadataBytes = metadata.getBytes(StandardCharsets.UTF_8).length;
+ long requiredBytes = frameSize(metadataBytes, 0) * 2;
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ requiredBytes += frameSize(metadataBytes, request.getBody().length);
+ }
+ if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ > segmentSizeBytes) {
+ sealSegment();
+ segmentId++;
+ openNewSegment();
+ }
+ }
+
+ private void rollBeforeControlIfNecessary(final String metadata) throws IOException {
+ final long requiredBytes = frameSize(metadata.getBytes(StandardCharsets.UTF_8).length, 0);
+ if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
+ > segmentSizeBytes) {
+ sealSegment();
+ segmentId++;
+ openNewSegment();
+ }
+ }
+
+ private static long frameSize(final long metadataBytes, final long payloadBytes) {
+ return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES;
+ }
+
+ private static String computeEventDigest(final List requests)
+ throws IOException {
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ for (final TPipeTransferReq request : requests) {
+ if (request == null || request.getBody() == null) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9);
+ }
+ updateEventDigest(digest, request.getVersion(), request.getType(), request.getBody());
+ }
+ return LogicalBackupFormat.toHex(digest.digest());
+ }
+
+ private static void updateEventDigest(
+ final MessageDigest digest,
+ final byte requestVersion,
+ final short requestType,
+ final byte[] body) {
+ digest.update(requestVersion);
+ digest.update(
+ ByteBuffer.allocate(Short.BYTES).order(ByteOrder.BIG_ENDIAN).putShort(requestType).array());
+ digest.update(body);
+ }
+
+ private void rollbackTo(final long position) throws IOException {
+ final Path segmentPath = currentSegmentPath();
+ channel.truncate(position);
+ channel.force(true);
+ channel.close();
+ channel = null;
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ openExistingSegment(segmentPath, scan);
+ rebuildManifestFromSegments();
+ manifest.recovered = true;
+ manifest.status = "RECOVERED";
+ manifest.lastDurableSequence = manifest.lastSequence;
+ operationsSinceFsync = 0;
+ lastFsyncAt = System.currentTimeMillis();
+ updateActiveSegmentManifest();
+ writeManifest();
+ }
+
+ private void rebuildManifestFromSegments() throws IOException {
+ manifest.firstSequence = -1;
+ manifest.lastSequence = -1;
+ manifest.lastEventGroupId = null;
+ manifest.lastEventDigest = null;
+ manifest.lastEventFirstSequence = -1;
+ manifest.skippedEventCount = 0;
+ manifest.operationCounts = new LinkedHashMap<>();
+
+ long expectedSequence = -1;
+ long expectedSegmentId = -1;
+ for (int segmentIndex = 0; segmentIndex < manifest.segments.size(); segmentIndex++) {
+ final LogicalBackupManifest.Segment segment = manifest.segments.get(segmentIndex);
+ final Path segmentPath = resolveExistingSegmentPath(segment.file);
+ final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false);
+ if (scan.getSegmentId() != segment.segmentId) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ if ((expectedSegmentId >= 0 && scan.getSegmentId() != expectedSegmentId)
+ || (!scan.isSealed() && segmentIndex != manifest.segments.size() - 1)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
+ segmentPath));
+ }
+ expectedSegmentId = scan.getSegmentId() + 1;
+ if (scan.isSealed() && "SEALED".equals(segment.status)) {
+ final LogicalBackupSegmentReader.Footer footer = scan.getFooter();
+ if (segment.firstSequence != footer.getFirstSequence()
+ || segment.lastSequence != footer.getLastSequence()
+ || segment.recordCount != footer.getRecordCount()
+ || segment.sizeBytes != scan.getValidLength()
+ || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3,
+ segmentPath));
+ }
+ }
+
+ segment.firstSequence = -1;
+ segment.lastSequence = -1;
+ segment.recordCount = scan.getRecords().size();
+ segment.sizeBytes = scan.getValidLength();
+ segment.status = scan.isSealed() ? "SEALED" : "ACTIVE";
+ segment.sha256 =
+ scan.isSealed() ? LogicalBackupFormat.toHex(scan.getFooter().getDigest()) : null;
+
+ MessageDigest eventDigest = null;
+ UUID eventGroupId = null;
+ long eventFirstSequence = -1;
+ for (final LogicalBackupRecord record : scan.getRecords()) {
+ if (expectedSequence >= 0 && record.getSequence() != expectedSequence) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149,
+ segmentPath));
+ }
+ expectedSequence = record.getSequence() + 1;
+ segment.firstSequence =
+ segment.firstSequence < 0 ? record.getSequence() : segment.firstSequence;
+ segment.lastSequence = record.getSequence();
+ manifest.firstSequence =
+ manifest.firstSequence < 0 ? record.getSequence() : manifest.firstSequence;
+ manifest.lastSequence = record.getSequence();
+ manifest.operationCounts.merge(record.getRecordType().name(), 1L, Long::sum);
+ if (record.getRecordType() == LogicalBackupRecordType.SKIPPED_EVENT) {
+ manifest.skippedEventCount++;
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) {
+ eventDigest = LogicalBackupFormat.newSha256();
+ eventGroupId = record.getEventGroupId();
+ eventFirstSequence = record.getSequence();
+ } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) {
+ updateEventDigest(
+ eventDigest,
+ record.getRequestVersion(),
+ record.getRequestType(),
+ record.getPayload());
+ } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) {
+ manifest.lastEventGroupId = eventGroupId.toString();
+ manifest.lastEventDigest = LogicalBackupFormat.toHex(eventDigest.digest());
+ manifest.lastEventFirstSequence = eventFirstSequence;
+ eventDigest = null;
+ eventGroupId = null;
+ eventFirstSequence = -1;
+ }
+ }
+ }
+ nextSequence = Math.max(0, manifest.lastSequence + 1);
+ streamEnded =
+ manifest.lastSequence >= 0
+ && !manifest.segments.isEmpty()
+ && isLastRecordStreamEnd(reader.scan(currentSegmentPath(), false).getRecords());
+ }
+
+ private static boolean isLastRecordStreamEnd(final List records) {
+ return !records.isEmpty()
+ && records.get(records.size() - 1).getRecordType() == LogicalBackupRecordType.STREAM_END;
+ }
+
+ private Path currentSegmentPath() throws IOException {
+ if (manifest.segments.isEmpty()) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15);
+ }
+ return resolveSegmentPath(manifest.segments.get(manifest.segments.size() - 1).file);
+ }
+
+ private Path resolveSegmentPath(final String file) throws IOException {
+ final Path relativePath;
+ try {
+ relativePath = file == null ? null : Path.of(file);
+ } catch (final RuntimeException e) {
+ throw invalidSegmentPath(file, e);
+ }
+ final Path normalizedDirectory = directory.toAbsolutePath().normalize();
+ if (relativePath == null || relativePath.isAbsolute() || relativePath.getNameCount() != 1) {
+ throw invalidSegmentPath(file, null);
+ }
+ final Path resolved = normalizedDirectory.resolve(relativePath).normalize();
+ if (!resolved.startsWith(normalizedDirectory)
+ || !normalizedDirectory.equals(resolved.getParent())) {
+ throw invalidSegmentPath(file, null);
+ }
+ return resolved;
+ }
+
+ private Path resolveExistingSegmentPath(final String file) throws IOException {
+ final Path resolved = resolveSegmentPath(file);
+ if (Files.isSymbolicLink(resolved)
+ || !Files.isRegularFile(resolved, LinkOption.NOFOLLOW_LINKS)) {
+ throw invalidSegmentPath(file, null);
+ }
+ return resolved;
+ }
+
+ private static IOException invalidSegmentPath(final String file, final Throwable cause) {
+ final IOException exception =
+ new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
+ file));
+ if (cause != null) {
+ exception.initCause(cause);
+ }
+ return exception;
+ }
+
+ private void updateActiveSegmentManifest() throws IOException {
+ if (channel == null || manifest.segments.isEmpty()) {
+ return;
+ }
+ final LogicalBackupManifest.Segment segment =
+ manifest.segments.get(manifest.segments.size() - 1);
+ segment.firstSequence = segmentFirstSequence;
+ segment.lastSequence = segmentLastSequence;
+ segment.recordCount = segmentRecordCount;
+ segment.sizeBytes = channel.position();
+ segment.status = "ACTIVE";
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
new file mode 100644
index 000000000000..05f6298e95fa
--- /dev/null
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupArchiveReaderTest {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final int MAX_RECORD_BYTES = 1024;
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data-stream", "data", 4096);
+ writeStream(root.resolve("schema"), "schema-stream", "schema", 4096);
+
+ final List streams = new LogicalBackupArchiveReader().read(root, false);
+
+ Assert.assertEquals(2, streams.size());
+ Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId);
+ Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId);
+ Assert.assertEquals(1, streams.get(0).getEventGroups().size());
+ Assert.assertEquals(4, streams.get(0).getRecords().size());
+ Assert.assertEquals(
+ request(1),
+ streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq());
+ }
+
+ @Test
+ public void testManifestCounterMismatchIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.operationCounts.clear();
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testUnlistedSegmentIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ Files.copy(onlySegment(directory), directory.resolve("unlisted.pwal"));
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ }
+
+ assertReadFails(directory, false);
+ final List streams = new LogicalBackupArchiveReader().read(directory, true);
+ Assert.assertEquals(1, streams.size());
+ Assert.assertEquals(1, streams.get(0).getEventGroups().size());
+ }
+
+ @Test
+ public void testUnsealedNonLastSegmentIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 500);
+ final List segments = segments(directory);
+ Assert.assertTrue(segments.size() > 1);
+ try (final RandomAccessFile file = new RandomAccessFile(segments.get(0).toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE);
+ }
+
+ assertReadFails(directory, true);
+ }
+
+ @Test
+ public void testParentSegmentPathIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "stream", "data", 4096);
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.segments.get(0).file = "../outside.pwal";
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testSkippedEventRequiresExplicitIncompleteOptIn() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = manifest("stream", "data");
+ try (final LogicalBackupWriter writer = writer(directory, manifest, 4096)) {
+ writer.recordSkippedEvent(1, "unsupported-event");
+ }
+
+ assertReadFails(directory, false);
+ Assert.assertEquals(1, new LogicalBackupArchiveReader().read(directory, true).size());
+
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest persistedManifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ persistedManifest.skippedEventCount = 0;
+ Files.writeString(manifestPath, GSON.toJson(persistedManifest), StandardCharsets.UTF_8);
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testMixedBackupSessionsAreRejected() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data-stream", "data", 4096);
+ writeStream(root.resolve("schema"), "schema-stream", "schema", 4096);
+ final Path manifestPath =
+ root.resolve("schema").resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(
+ Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class);
+ manifest.backupId = "another-backup";
+ Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8);
+
+ assertReadFails(root, false);
+ }
+
+ @Test
+ public void testNonReplayableRequestTypeIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) {
+ writer.writeEvent(
+ UUID.randomUUID(),
+ 1,
+ Collections.singletonList(
+ new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 200)
+ .setBody(ByteBuffer.wrap(new byte[] {1}))),
+ "metadata");
+ }
+
+ assertReadFails(directory, false);
+ }
+
+ @Test
+ public void testDuplicateEventGroupIdIsRejected() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventGroupId = UUID.randomUUID();
+ try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) {
+ writer.writeEvent(eventGroupId, 1, Collections.singletonList(request(1)), "metadata");
+ writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "metadata");
+ writer.writeEvent(eventGroupId, 3, Collections.singletonList(request(1)), "metadata");
+ }
+
+ assertReadFails(directory, false);
+ }
+
+ private static void writeStream(
+ final Path directory, final String streamId, final String streamType, final long segmentSize)
+ throws Exception {
+ try (final LogicalBackupWriter writer =
+ writer(directory, manifest(streamId, streamType), segmentSize)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata");
+ if (segmentSize < 1000) {
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "metadata");
+ }
+ }
+ }
+
+ private static LogicalBackupManifest manifest(final String streamId, final String streamType) {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = streamId;
+ manifest.streamType = streamType;
+ manifest.timestampPrecision = "ms";
+ return manifest;
+ }
+
+ private static LogicalBackupWriter writer(
+ final Path directory, final LogicalBackupManifest manifest, final long segmentSize)
+ throws IOException {
+ return new LogicalBackupWriter(
+ directory,
+ manifest,
+ segmentSize,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ false);
+ }
+
+ private static TPipeTransferReq request(final int value) {
+ return request(new byte[] {(byte) value});
+ }
+
+ private static TPipeTransferReq request(final byte[] body) {
+ return new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 10)
+ .setBody(ByteBuffer.wrap(body));
+ }
+
+ private static Path onlySegment(final Path directory) throws IOException {
+ final List segments = segments(directory);
+ Assert.assertEquals(1, segments.size());
+ return segments.get(0);
+ }
+
+ private static List segments(final Path directory) throws IOException {
+ try (final java.util.stream.Stream files = Files.list(directory)) {
+ return files
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .sorted()
+ .collect(java.util.stream.Collectors.toList());
+ }
+ }
+
+ private static void assertReadFails(final Path source, final boolean allowIncomplete)
+ throws IOException {
+ try {
+ new LogicalBackupArchiveReader().read(source, allowIncomplete);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
new file mode 100644
index 000000000000..b6806f3407be
--- /dev/null
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+public class LogicalBackupWriterTest {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final int MAX_RECORD_BYTES = 1024;
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testRoundTripAndDuplicateEvent() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventId = UUID.randomUUID();
+ final TPipeTransferReq request = request((byte) 1, (short) 2, new byte[] {3, 4, 5});
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m"));
+ Assert.assertEquals(
+ 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m"));
+ }
+
+ final LogicalBackupSegmentReader.ScanResult result = scanOnlySegment(directory);
+ Assert.assertTrue(result.isSealed());
+ Assert.assertEquals(4, result.getRecords().size());
+ Assert.assertEquals(
+ LogicalBackupRecordType.EVENT_BEGIN, result.getRecords().get(0).getRecordType());
+ Assert.assertEquals(
+ LogicalBackupRecordType.PIPE_REQUEST, result.getRecords().get(1).getRecordType());
+ Assert.assertEquals(request, result.getRecords().get(1).toTPipeTransferReq());
+ Assert.assertEquals(
+ LogicalBackupRecordType.EVENT_COMMIT, result.getRecords().get(2).getRecordType());
+ Assert.assertEquals(
+ LogicalBackupRecordType.STREAM_END, result.getRecords().get(3).getRecordType());
+ Assert.assertEquals(0, result.getFooter().getFirstSequence());
+ Assert.assertEquals(3, result.getFooter().getLastSequence());
+ }
+
+ @Test
+ public void testDuplicateEventWithDifferentDigestFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID eventId = UUID.randomUUID();
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(eventId, 1, Collections.singletonList(request(1)), "");
+ try {
+ writer.writeEvent(eventId, 1, Collections.singletonList(request(2)), "");
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains(eventId.toString()));
+ }
+ }
+ }
+
+ @Test
+ public void testOnlineRollbackAfterPartialEvent() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ final List requests =
+ new AbstractList() {
+ private int accesses;
+
+ @Override
+ public TPipeTransferReq get(final int index) {
+ if (++accesses == 6) {
+ throw new IllegalStateException();
+ }
+ return request(index + 1);
+ }
+
+ @Override
+ public int size() {
+ return 2;
+ }
+ };
+ try {
+ writer.writeEvent(UUID.randomUUID(), 1, requests, "");
+ Assert.fail();
+ } catch (final IllegalStateException expected) {
+ // Expected.
+ }
+ Assert.assertEquals(
+ 0,
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(3)), "after-rollback"));
+ }
+
+ final List records = scanOnlySegment(directory).getRecords();
+ Assert.assertEquals(4, records.size());
+ Assert.assertEquals(0, records.get(0).getSequence());
+ Assert.assertEquals("after-rollback", records.get(0).getMetadata());
+ }
+
+ @Test
+ public void testRecoverIncompleteEventAndRebuildCounts() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ final long beginAndRequestLength =
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE + frameSize(0, 0) + frameSize(0, 1);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(beginAndRequestLength);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_BEGIN"));
+ Assert.assertEquals(
+ Long.valueOf(1), writer.getManifest().operationCounts.get("PIPE_REQUEST"));
+ Assert.assertEquals(
+ Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_COMMIT"));
+ }
+ }
+
+ @Test
+ public void testRecoverIncompleteFrame() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 2);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(2, writer.getManifest().lastSequence);
+ }
+ }
+
+ @Test
+ public void testRecoverPartialFooter() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE + 8);
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ Assert.assertEquals(
+ 4, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ }
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES)
+ .scan(segments(directory).get(segments(directory).size() - 1), false)
+ .isSealed());
+ }
+
+ @Test
+ public void testRecoverCorruptFooterWhenManifestStillMarksSegmentActive() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long lastByteOffset = file.length() - 1;
+ file.seek(lastByteOffset);
+ final int lastByte = file.readUnsignedByte();
+ file.seek(lastByteOffset);
+ file.writeByte(lastByte ^ 1);
+ }
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class);
+ manifest.status = "WRITING";
+ manifest.closedAt = null;
+ manifest.segments.get(0).status = "ACTIVE";
+ manifest.segments.get(0).sizeBytes -= LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ manifest.segments.get(0).sha256 = null;
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertTrue(writer.getManifest().recovered);
+ }
+ Assert.assertTrue(scanOnlySegment(directory).isSealed());
+ }
+
+ @Test
+ public void testChecksumDamageFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long payloadOffset =
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE
+ + frameSize("metadata".length(), 0)
+ + LogicalBackupFormat.RECORD_HEADER_SIZE
+ + "metadata".length();
+ file.seek(payloadOffset);
+ file.writeByte(file.readByte() ^ 1);
+ }
+ try {
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("CRC"));
+ }
+ }
+
+ @Test
+ public void testFooterSequenceDamageFails() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ }
+ final Path segment = onlySegment(directory);
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ final long footerOffset = file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE;
+ final byte[] footer = new byte[LogicalBackupFormat.SEGMENT_FOOTER_SIZE];
+ file.seek(footerOffset);
+ file.readFully(footer);
+ ByteBuffer.wrap(footer).order(ByteOrder.BIG_ENDIAN).putLong(Integer.BYTES + Long.BYTES, 99);
+ ByteBuffer.wrap(footer)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES,
+ LogicalBackupFormat.crc32c(
+ footer, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES));
+ file.seek(footerOffset);
+ file.write(footer);
+ }
+ try {
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("metadata"));
+ }
+ }
+
+ @Test
+ public void testSegmentRollover() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 500, false)) {
+ writer.writeEvent(
+ UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m");
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m");
+ }
+ final List segments = segments(directory);
+ Assert.assertEquals(2, segments.size());
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(0), false).isSealed());
+ Assert.assertTrue(
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed());
+ }
+
+ @Test
+ public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = manifest();
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 500,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.NONE,
+ 1,
+ 1,
+ false)) {
+ writer.writeEvent(
+ UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m");
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m");
+ Assert.assertEquals(2, writer.getManifest().lastDurableSequence);
+ }
+ }
+
+ @Test
+ public void testControlRecordRollover() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer = writer(directory, 258, false)) {
+ writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first");
+ writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second");
+ Assert.assertEquals(2, writer.getManifest().segments.size());
+ Assert.assertEquals(1, writer.getManifest().lastSequence);
+ }
+ }
+
+ @Test
+ public void testDirectoryLock() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("locked"));
+ }
+ }
+ }
+
+ @Test
+ public void testFailIfDirectoryExists() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath();
+ try {
+ writer(directory, 4096, false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains(directory.toString()));
+ }
+ }
+
+ @Test
+ public void testAppendRejectsConfigurationMismatch() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create the backup.
+ }
+ try {
+ writer(directory, 8192, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("segmentSizeBytes"));
+ }
+ }
+
+ @Test
+ public void testAppendRejectsParentSegmentPath() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ final Path directory = root.resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create the backup.
+ }
+ final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class);
+ final String originalSegment = manifest.segments.get(0).file;
+ Files.copy(directory.resolve(originalSegment), root.resolve("outside.pwal"));
+ manifest.segments.get(0).file = "../outside.pwal";
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("segment path"));
+ }
+
+ manifest.segments.get(0).file = originalSegment;
+ Files.writeString(manifestPath, GSON.toJson(manifest));
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, true)) {
+ // A failed append must release the directory lock.
+ }
+ }
+
+ @Test
+ public void testAppendAdoptsOrphanHeaderOnlySegmentAndClearsClosedState() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path orphan = directory.resolve(segmentFileName(1));
+ Files.write(orphan, segmentHeader(1, 123));
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals("WRITING", writer.getManifest().status);
+ Assert.assertNull(writer.getManifest().closedAt);
+ Assert.assertEquals(2, writer.getManifest().segments.size());
+ Assert.assertEquals(
+ 1, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), ""));
+ }
+
+ final LogicalBackupSegmentReader.ScanResult scan =
+ new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(orphan, false);
+ Assert.assertTrue(scan.isSealed());
+ Assert.assertEquals(123, scan.getCreatedAt());
+ }
+
+ @Test
+ public void testAppendRejectsNonEmptyOrphanSegment() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path orphan = directory.resolve(segmentFileName(1));
+ final byte[] invalidOrphan =
+ Arrays.copyOf(segmentHeader(1, 123), LogicalBackupFormat.SEGMENT_HEADER_SIZE + 1);
+ Files.write(orphan, invalidOrphan);
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // The extra byte is an incomplete record tail and must never be adopted.
+ }
+ }
+
+ @Test
+ public void testAppendRejectsUnexpectedUnlistedSegment() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) {
+ // Create and seal the initial segment.
+ }
+
+ final Path unexpected = directory.resolve(segmentFileName(99));
+ Files.write(unexpected, segmentHeader(99, 123));
+ try {
+ writer(directory, 4096, true);
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertTrue(expected.getMessage().contains("Unlisted"));
+ }
+ }
+
+ @Test
+ public void testPeriodicFsyncAdvancesOnHeartbeat() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "stream";
+ manifest.streamType = "data";
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 4096,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.PERIODIC,
+ 1000,
+ 100,
+ false)) {
+ writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "");
+ Assert.assertEquals(-1, writer.getManifest().lastDurableSequence);
+ Thread.sleep(150);
+ writer.heartbeat();
+ Assert.assertEquals(
+ writer.getManifest().lastSequence, writer.getManifest().lastDurableSequence);
+ }
+ }
+
+ @Test
+ public void testMaxRecordBytesHasHardLimit() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ try {
+ new LogicalBackupWriter(
+ directory,
+ manifest,
+ 4096,
+ LogicalBackupFormat.MAX_RECORD_BYTES + 1,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Expected.
+ }
+ }
+
+ private static LogicalBackupWriter writer(
+ final Path directory, final long segmentSize, final boolean append) throws IOException {
+ return new LogicalBackupWriter(
+ directory,
+ manifest(),
+ segmentSize,
+ MAX_RECORD_BYTES,
+ LogicalBackupWriter.FsyncPolicy.ALWAYS,
+ 1,
+ 1,
+ append);
+ }
+
+ private static LogicalBackupManifest manifest() {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "stream";
+ manifest.streamType = "data";
+ return manifest;
+ }
+
+ private static TPipeTransferReq request(final int value) {
+ return request(new byte[] {(byte) value});
+ }
+
+ private static TPipeTransferReq request(final byte[] body) {
+ return request((byte) 1, (short) 2, body);
+ }
+
+ private static TPipeTransferReq request(final byte version, final short type, final byte[] body) {
+ return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body));
+ }
+
+ private static LogicalBackupSegmentReader.ScanResult scanOnlySegment(final Path directory)
+ throws IOException {
+ return new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(onlySegment(directory), false);
+ }
+
+ private static Path onlySegment(final Path directory) throws IOException {
+ final List segments = segments(directory);
+ Assert.assertEquals(1, segments.size());
+ return segments.get(0);
+ }
+
+ private static List segments(final Path directory) throws IOException {
+ try (final java.util.stream.Stream files = Files.list(directory)) {
+ final Path[] segments =
+ files
+ .filter(path -> path.getFileName().toString().endsWith(".pwal"))
+ .sorted()
+ .toArray(Path[]::new);
+ return Arrays.asList(segments);
+ }
+ }
+
+ private static long frameSize(final int metadataBytes, final int payloadBytes) {
+ return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES;
+ }
+
+ private static String segmentFileName(final long segmentId) {
+ return String.format(java.util.Locale.ROOT, "segment-%020d.pwal", segmentId);
+ }
+
+ private static byte[] segmentHeader(final long segmentId, final long createdAt) {
+ final ByteBuffer header =
+ ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN);
+ header.putLong(LogicalBackupFormat.SEGMENT_MAGIC);
+ header.putShort(LogicalBackupFormat.MAJOR_VERSION);
+ header.putShort(LogicalBackupFormat.MINOR_VERSION);
+ header.putLong(segmentId);
+ header.putLong(createdAt);
+ header.putInt(0);
+ final byte[] bytes = header.array();
+ ByteBuffer.wrap(bytes)
+ .order(ByteOrder.BIG_ENDIAN)
+ .putInt(
+ LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES,
+ LogicalBackupFormat.crc32c(
+ bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES));
+ return bytes;
+ }
+}
diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-pipe-logical-backup.sh
new file mode 100644
index 000000000000..8a0a4f8a1834
--- /dev/null
+++ b/scripts/tools/export-pipe-logical-backup.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@"
diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-pipe-logical-backup.sh
new file mode 100644
index 000000000000..6e4e069210de
--- /dev/null
+++ b/scripts/tools/import-pipe-logical-backup.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@"
diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/pipe-logical-backup.sh
new file mode 100644
index 000000000000..a0f6a3529f74
--- /dev/null
+++ b/scripts/tools/pipe-logical-backup.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -n "${IOTDB_INCLUDE}" ] && [ -r "${IOTDB_INCLUDE}" ]; then
+ . "${IOTDB_INCLUDE}"
+fi
+
+if [ -z "${IOTDB_HOME}" ]; then
+ IOTDB_HOME="$(cd "$(dirname "$0")/.."; pwd)"
+ export IOTDB_HOME
+fi
+
+if [ -n "${JAVA_HOME}" ] && [ -x "${JAVA_HOME}/bin/java" ]; then
+ JAVA="${JAVA_HOME}/bin/java"
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for jar in "${IOTDB_HOME}"/lib/*.jar; do
+ CLASSPATH="${CLASSPATH}:${jar}"
+done
+
+exec "${JAVA}" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 \
+ -DIOTDB_HOME="${IOTDB_HOME}" -cp "${CLASSPATH}" \
+ org.apache.iotdb.tool.pipe.PipeLogicalBackupTool "$@"
diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-pipe-logical-backup.bat
new file mode 100644
index 000000000000..42e1df8afb27
--- /dev/null
+++ b/scripts/tools/windows/export-pipe-logical-backup.bat
@@ -0,0 +1,21 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@call "%~dp0pipe-logical-backup.bat" export %*
+@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-pipe-logical-backup.bat
new file mode 100644
index 000000000000..0fba332d5a2e
--- /dev/null
+++ b/scripts/tools/windows/import-pipe-logical-backup.bat
@@ -0,0 +1,21 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@call "%~dp0pipe-logical-backup.bat" import %*
+@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/pipe-logical-backup.bat
new file mode 100644
index 000000000000..14e3180e4699
--- /dev/null
+++ b/scripts/tools/windows/pipe-logical-backup.bat
@@ -0,0 +1,40 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..\..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED JAVA_HOME goto :no_java
+set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*"
+"%JAVA_HOME%\bin\java" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 ^
+ -DIOTDB_HOME="%IOTDB_HOME%" -cp %CLASSPATH% ^
+ org.apache.iotdb.tool.pipe.PipeLogicalBackupTool %*
+set ret_code=%ERRORLEVEL%
+goto :finally
+
+:no_java
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+
+:finally
+endlocal & exit /B %ret_code%
From ce4b30d0ea26d433ca780e03fb691100ab15c268 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Fri, 11 Sep 2026 12:03:37 +0800
Subject: [PATCH 2/8] Fix logical backup tablet serialization error handling
---
.../logicalbackup/LogicalBackupSink.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
index 5050107c05f9..594d49922f7e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -51,6 +51,7 @@
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
@@ -389,13 +390,17 @@ public void transfer(final TsFileInsertionEvent event) throws Exception {
try {
tsFile.consumeTabletInsertionEventsWithRetry(
tablet -> {
- requests.add(
- PipeTransferTabletRawReqV2.toTPipeTransferReq(
- tablet.convertToTablet(),
- tablet.isAligned(),
- tablet.isTableModelEvent()
- ? tablet.getTableModelDatabaseName()
- : tablet.getTreeModelDatabaseName()));
+ try {
+ requests.add(
+ PipeTransferTabletRawReqV2.toTPipeTransferReq(
+ tablet.convertToTablet(),
+ tablet.isAligned(),
+ tablet.isTableModelEvent()
+ ? tablet.getTableModelDatabaseName()
+ : tablet.getTreeModelDatabaseName()));
+ } catch (final IOException e) {
+ throw new PipeException(e.getMessage(), e);
+ }
},
getClass().getName());
if (!requests.isEmpty()) {
From 218dcb43001c4f99f5905d0cba032ee7e62478c1 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Wed, 16 Sep 2026 12:06:00 +0800
Subject: [PATCH 3/8] Harden logical backup failover and import recovery
---
.../apache/iotdb/cli/i18n/CliMessages.java | 5 +-
.../apache/iotdb/cli/i18n/CliMessages.java | 5 +-
.../tool/pipe/PipeLogicalBackupTool.java | 191 +++++++++++++-----
.../tool/pipe/PipeLogicalBackupToolTest.java | 111 +++++++++-
.../iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../logicalbackup/LogicalBackupSink.java | 25 +++
.../logicalbackup/LogicalBackupSinkTest.java | 79 ++++++++
.../LogicalBackupWriterTest.java | 30 +++
9 files changed, 390 insertions(+), 62 deletions(-)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 77888e65768a..8aa2feccc907 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -130,9 +130,8 @@ private CliMessages() {}
public static final String
LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
"Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
- public static final String
- LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 =
- "pipe-logical-backup ";
+ public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
+ "pipe-logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
"Logical backup archive entry is unsafe: %s";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index 530b504b29ce..35346133f19e 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -126,9 +126,8 @@ private CliMessages() {}
public static final String
LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
"使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
- public static final String
- LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 =
- "pipe-logical-backup ";
+ public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
+ "pipe-logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
"逻辑备份归档项路径不安全：%s";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
index 5d80243f3adf..1d2315f17c04 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -80,8 +80,6 @@ public final class PipeLogicalBackupTool {
private static final String COMMAND_VERIFY = "verify";
private static final String COMMAND_EXPORT = "export";
private static final String COMMAND_IMPORT = "import";
- private static final String COMMAND_RESTORE = "restore";
- private static final String COMMAND_STATS = "stats";
private static final String OPTION_INPUT = "input";
private static final String OPTION_OUTPUT = "output";
private static final String OPTION_RESUME = "resume";
@@ -137,10 +135,7 @@ static int run(final String[] args) throws Exception {
case COMMAND_EXPORT:
return export(line);
case COMMAND_IMPORT:
- case COMMAND_RESTORE:
return importBackup(line);
- case COMMAND_STATS:
- return inspect(line);
default:
throw new ParseException(
String.format(
@@ -352,19 +347,13 @@ private static int importBackup(final CommandLine line) throws Exception {
handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
long importedGroups = 0;
for (final BackupStream stream : streams) {
- final String streamId = stream.getManifest().streamId;
- final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L);
- for (final EventGroup group : stream.getEventGroups()) {
- if (group.getLastSequence() <= lastApplied) {
- continue;
- }
- for (final LogicalBackupRecord record : group.getRequests()) {
- transfer(client, record.toTPipeTransferReq());
- }
- checkpointState.appliedSequences.put(streamId, group.getLastSequence());
- writeCheckpoint(checkpoint, checkpointState);
- importedGroups++;
- }
+ importedGroups +=
+ importStream(
+ checkpoint,
+ checkpointState,
+ stream,
+ request -> transfer(client, request),
+ state -> writeCheckpoint(checkpoint, state));
}
writeCheckpoint(checkpoint, checkpointState);
System.out.println(
@@ -377,6 +366,49 @@ private static int importBackup(final CommandLine line) throws Exception {
return 0;
}
+ static long importStream(
+ final Path checkpoint,
+ final ImportCheckpoint checkpointState,
+ final BackupStream stream,
+ final RequestTransfer requestTransfer,
+ final CheckpointPersister checkpointPersister)
+ throws Exception {
+ final String streamId = stream.getManifest().streamId;
+ final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L);
+ long importedGroups = 0;
+ for (final EventGroup group : stream.getEventGroups()) {
+ if (group.getLastSequence() <= lastApplied) {
+ continue;
+ }
+
+ int nextRequestIndex = 0;
+ if (checkpointState.inProgress == null) {
+ checkpointState.inProgress = ImportProgress.start(streamId, group);
+ checkpointPersister.persist(checkpointState);
+ } else if (checkpointState.inProgress.matches(streamId, group)) {
+ nextRequestIndex = checkpointState.inProgress.nextRequestIndex;
+ } else {
+ throw invalidCheckpoint(checkpoint);
+ }
+
+ final List requests = group.getRequests();
+ if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) {
+ throw invalidCheckpoint(checkpoint);
+ }
+ for (int index = nextRequestIndex; index < requests.size(); index++) {
+ requestTransfer.transfer(requests.get(index).toTPipeTransferReq());
+ checkpointState.inProgress.nextRequestIndex = index + 1;
+ checkpointPersister.persist(checkpointState);
+ }
+
+ checkpointState.appliedSequences.put(streamId, group.getLastSequence());
+ checkpointState.inProgress = null;
+ checkpointPersister.persist(checkpointState);
+ importedGroups++;
+ }
+ return importedGroups;
+ }
+
private static List read(final CommandLine line) throws IOException {
final String source = input(line);
try (final BackupInput input = BackupInput.open(Path.of(source))) {
@@ -472,14 +504,15 @@ private static void handshake(
}
}
- private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
+ static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
throws TException, IOException {
final TPipeTransferResp response = client.pipeTransfer(request);
if (response == null
|| response.getStatus() == null
|| (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
&& response.getStatus().getCode()
- != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) {
+ != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) {
throw new IOException(
String.format(
CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11,
@@ -497,7 +530,7 @@ private static Path checkpointPath(final CommandLine line) throws IOException {
return source.resolveSibling(source.getFileName() + ".import.checkpoint.json");
}
- private static ImportCheckpoint readCheckpoint(
+ static ImportCheckpoint readCheckpoint(
final Path checkpoint,
final List streams,
final String host,
@@ -539,7 +572,7 @@ private static ImportCheckpoint readCheckpoint(
|| !host.equals(state.targetHost)
|| port != state.targetPort
|| !user.equals(state.targetUser)
- || !checkpointSequencesAreValid(streams, state.appliedSequences)) {
+ || !checkpointStateIsValid(streams, state)) {
throw new IOException(
String.format(
CliMessages
@@ -615,34 +648,66 @@ private static void updateDigest(final MessageDigest digest, final long value) {
digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array());
}
- private static boolean checkpointSequencesAreValid(
- final List streams, final Map appliedSequences) {
+ private static boolean checkpointStateIsValid(
+ final List streams, final ImportCheckpoint state) {
+ final Map appliedSequences = state.appliedSequences;
if (appliedSequences.size() > streams.size()) {
return false;
}
+ boolean foundIncompleteStream = false;
+ boolean matchedInProgress = false;
for (final BackupStream stream : streams) {
- if (!appliedSequences.containsKey(stream.getManifest().streamId)) {
- continue;
- }
- final Long applied = appliedSequences.get(stream.getManifest().streamId);
- if (applied == null) {
+ final String streamId = stream.getManifest().streamId;
+ final Long applied = appliedSequences.getOrDefault(streamId, -1L);
+ if (applied == null || applied < -1) {
return false;
}
- if (applied == -1) {
- continue;
+
+ int appliedGroupIndex = -1;
+ if (applied != -1) {
+ for (int index = 0; index < stream.getEventGroups().size(); index++) {
+ if (stream.getEventGroups().get(index).getLastSequence() == applied) {
+ appliedGroupIndex = index;
+ break;
+ }
+ }
+ if (appliedGroupIndex < 0) {
+ return false;
+ }
}
- if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) {
+
+ final boolean streamComplete = appliedGroupIndex == stream.getEventGroups().size() - 1;
+ if (foundIncompleteStream && appliedGroupIndex >= 0) {
return false;
}
+ if (!streamComplete && !foundIncompleteStream) {
+ foundIncompleteStream = true;
+ if (state.inProgress != null) {
+ final EventGroup nextGroup = stream.getEventGroups().get(appliedGroupIndex + 1);
+ if (!state.inProgress.matches(streamId, nextGroup)
+ || state.inProgress.nextRequestIndex < 0
+ || state.inProgress.nextRequestIndex > nextGroup.getRequests().size()) {
+ return false;
+ }
+ matchedInProgress = true;
+ }
+ }
}
- return appliedSequences.keySet().stream()
- .allMatch(
- streamId ->
- streams.stream()
- .anyMatch(stream -> stream.getManifest().streamId.equals(streamId)));
+ return (state.inProgress == null || matchedInProgress)
+ && appliedSequences.keySet().stream()
+ .allMatch(
+ streamId ->
+ streams.stream()
+ .anyMatch(stream -> stream.getManifest().streamId.equals(streamId)));
+ }
+
+ private static IOException invalidCheckpoint(final Path checkpoint) {
+ return new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint));
}
- private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState)
+ static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState)
throws IOException {
final Path parent = checkpoint.toAbsolutePath().normalize().getParent();
if (parent != null) {
@@ -681,7 +746,7 @@ private static void printUsage() {
formatter.printHelp(
new PrintWriter(System.out),
120,
- CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2,
+ CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE,
CliMessages
.LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E,
optionsForHelp(),
@@ -710,14 +775,46 @@ private static Options optionsForHelp() {
return options;
}
- private static class ImportCheckpoint {
- private String formatName;
- private String formatVersion;
- private Map sourceStreams = new LinkedHashMap<>();
- private String targetHost;
- private int targetPort;
- private String targetUser;
- private Map appliedSequences = new LinkedHashMap<>();
+ static class ImportCheckpoint {
+ String formatName;
+ String formatVersion;
+ Map sourceStreams = new LinkedHashMap<>();
+ String targetHost;
+ int targetPort;
+ String targetUser;
+ Map appliedSequences = new LinkedHashMap<>();
+ ImportProgress inProgress;
+ }
+
+ static class ImportProgress {
+ String streamId;
+ String eventGroupId;
+ long lastSequence;
+ int nextRequestIndex;
+
+ static ImportProgress start(final String streamId, final EventGroup group) {
+ final ImportProgress progress = new ImportProgress();
+ progress.streamId = streamId;
+ progress.eventGroupId = group.getEventGroupId().toString();
+ progress.lastSequence = group.getLastSequence();
+ return progress;
+ }
+
+ boolean matches(final String candidateStreamId, final EventGroup group) {
+ return candidateStreamId.equals(streamId)
+ && group.getEventGroupId().toString().equals(eventGroupId)
+ && group.getLastSequence() == lastSequence;
+ }
+ }
+
+ @FunctionalInterface
+ interface RequestTransfer {
+ void transfer(TPipeTransferReq request) throws Exception;
+ }
+
+ @FunctionalInterface
+ interface CheckpointPersister {
+ void persist(ImportCheckpoint state) throws IOException;
}
private static class BackupInput implements AutoCloseable {
diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
index 0fe45b22d897..6e7af8726ab3 100644
--- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
+++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
@@ -19,27 +19,35 @@
package org.apache.iotdb.tool.pipe;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
import org.apache.commons.cli.ParseException;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
@@ -141,7 +149,93 @@ public void testPlaintextPasswordOptionIsRejected() throws Exception {
}
}
+ @Test
+ public void testRemovedCommandAliasesAreRejected() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source);
+
+ for (final String command : Arrays.asList("restore", "stats")) {
+ try {
+ PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()});
+ Assert.fail();
+ } catch (final ParseException expected) {
+ // Expected.
+ }
+ }
+ }
+
+ @Test
+ public void testTransferAcceptsIdempotentConflict() throws Exception {
+ final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class);
+ final TPipeTransferReq request = request(1);
+ Mockito.when(client.pipeTransfer(request))
+ .thenReturn(
+ new TPipeTransferResp(
+ new TSStatus(
+ TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())));
+
+ PipeLogicalBackupTool.transfer(client, request);
+ }
+
+ @Test
+ public void testImportCheckpointResumesAtRequestGranularity() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("source");
+ writeBackup(source, Arrays.asList(request(1), request(2)));
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ final BackupStream stream = streams.get(0);
+ final Path checkpoint = root.resolve("import.checkpoint.json");
+ final PipeLogicalBackupTool.ImportCheckpoint state =
+ PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root");
+ final List transferredBodies = new ArrayList<>();
+ final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true);
+
+ try {
+ PipeLogicalBackupTool.importStream(
+ checkpoint,
+ state,
+ stream,
+ request -> transferredBodies.add((int) request.getBody()[0]),
+ checkpointState -> {
+ if (checkpointState.inProgress != null
+ && checkpointState.inProgress.nextRequestIndex == 1
+ && failCheckpointOnce.getAndSet(false)) {
+ throw new IOException("simulated checkpoint failure");
+ }
+ PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState);
+ });
+ Assert.fail();
+ } catch (final IOException expected) {
+ // The request succeeded, but advancing its checkpoint was interrupted.
+ }
+
+ Assert.assertEquals(Collections.singletonList(1), transferredBodies);
+ final PipeLogicalBackupTool.ImportCheckpoint durableState =
+ PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root");
+ Assert.assertNotNull(durableState.inProgress);
+ Assert.assertEquals(0, durableState.inProgress.nextRequestIndex);
+
+ Assert.assertEquals(
+ 1,
+ PipeLogicalBackupTool.importStream(
+ checkpoint,
+ durableState,
+ stream,
+ request -> transferredBodies.add((int) request.getBody()[0]),
+ checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState)));
+ Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies);
+ Assert.assertNull(durableState.inProgress);
+ Assert.assertEquals(
+ Long.valueOf(stream.getEventGroups().get(0).getLastSequence()),
+ durableState.appliedSequences.get(stream.getManifest().streamId));
+ }
+
private static void writeBackup(final Path directory) throws Exception {
+ writeBackup(directory, Collections.singletonList(request(3)));
+ }
+
+ private static void writeBackup(final Path directory, final List requests)
+ throws Exception {
final LogicalBackupManifest manifest = new LogicalBackupManifest();
manifest.backupId = "backup";
manifest.pipeName = "pipe";
@@ -152,15 +246,14 @@ private static void writeBackup(final Path directory) throws Exception {
try (final LogicalBackupWriter writer =
new LogicalBackupWriter(
directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) {
- writer.writeEvent(
- UUID.randomUUID(),
- 1,
- Collections.singletonList(
- new TPipeTransferReq()
- .setVersion((byte) 1)
- .setType((short) 10)
- .setBody(ByteBuffer.wrap(new byte[] {3}))),
- "metadata");
+ writer.writeEvent(UUID.randomUUID(), 1, requests, "metadata");
}
}
+
+ private static TPipeTransferReq request(final int value) {
+ return new TPipeTransferReq()
+ .setVersion((byte) 1)
+ .setType((short) 10)
+ .setBody(ByteBuffer.wrap(new byte[] {(byte) value}));
+ }
}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index d530fc7cf1bd..3a914bc80c76 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2647,6 +2647,9 @@ private DataNodePipeMessages() {}
"Unsupported logical backup event policy: %s";
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
"Unsupported logical backup resume policy: %s";
+ public static final String
+ EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
+ "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory";
public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
"Unsupported Pipe event for logical backup: %s";
public static final String
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 49575e46abb9..08803638ea60 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2471,6 +2471,9 @@ private DataNodePipeMessages() {}
"不支持的逻辑备份事件策略：%s";
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
"不支持的逻辑备份恢复策略：%s";
+ public static final String
+ EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
+ "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录";
public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
"逻辑备份不支持 Pipe 事件：%s";
public static final String
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
index 594d49922f7e..ef152b58b7c3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -160,6 +160,7 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
+ validateDataRegionResumePolicy(parameters);
final String policy =
parameters
.getStringOrDefault(
@@ -187,6 +188,29 @@ private static void validateSynonymAttributes(
false);
}
+ private void validateDataRegionResumePolicy(final PipeParameters parameters)
+ throws PipeParameterNotValidException {
+ if (!"data".equals(streamType)) {
+ return;
+ }
+ final boolean explicitlyConfigured =
+ parameters.hasAnyAttributes(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ final String resume =
+ parameters.getStringOrDefault(
+ Arrays.asList(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE);
+ if (!explicitlyConfigured
+ || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) {
+ throw new PipeParameterNotValidException(
+ DataNodePipeMessages
+ .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8);
+ }
+ }
+
@Override
public void customize(
final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
@@ -253,6 +277,7 @@ public void customize(
PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE)
.toLowerCase(Locale.ROOT);
+ validateDataRegionResumePolicy(parameters);
final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume);
if (!append
&& !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
new file mode 100644
index 000000000000..37c3d622b4f5
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup;
+
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LogicalBackupSinkTest {
+
+ @Test
+ public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup");
+
+ assertInvalid(new LogicalBackupSink("data"), attributes);
+
+ attributes.put(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS);
+ assertInvalid(new LogicalBackupSink("data"), attributes);
+
+ attributes.put(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND);
+ new LogicalBackupSink("data")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+
+ attributes.remove(PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY);
+ attributes.put(
+ PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND);
+ new LogicalBackupSink("data")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+ }
+
+ @Test
+ public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup");
+
+ new LogicalBackupSink("schema")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+ }
+
+ private static void assertInvalid(
+ final LogicalBackupSink sink, final Map attributes) throws Exception {
+ try {
+ sink.validate(new PipeParameterValidator(new PipeParameters(new HashMap<>(attributes))));
+ Assert.fail();
+ } catch (final PipeParameterNotValidException expected) {
+ Assert.assertTrue(expected.getMessage().contains("append"));
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
index b6806f3407be..ec7d5d63ed9f 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
@@ -91,6 +91,31 @@ public void testDuplicateEventWithDifferentDigestFails() throws Exception {
}
}
+ @Test
+ public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ final UUID firstEventId = UUID.randomUUID();
+ final TPipeTransferReq firstRequest = replayableRequest(1);
+ try (final LogicalBackupWriter writer = writer(directory, 4096, false)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first"));
+ }
+
+ try (final LogicalBackupWriter writer = writer(directory, 4096, true)) {
+ Assert.assertEquals(
+ 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first"));
+ Assert.assertEquals(
+ 4,
+ writer.writeEvent(
+ UUID.randomUUID(), 2, Collections.singletonList(replayableRequest(2)), "second"));
+ }
+
+ final List streams =
+ new LogicalBackupArchiveReader().read(directory, false);
+ Assert.assertEquals(1, streams.size());
+ Assert.assertEquals(2, streams.get(0).getEventGroups().size());
+ }
+
@Test
public void testOnlineRollbackAfterPartialEvent() throws Exception {
final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
@@ -522,6 +547,7 @@ private static LogicalBackupManifest manifest() {
manifest.pipeCreationTime = 1;
manifest.streamId = "stream";
manifest.streamType = "data";
+ manifest.timestampPrecision = "ms";
return manifest;
}
@@ -533,6 +559,10 @@ private static TPipeTransferReq request(final byte[] body) {
return request((byte) 1, (short) 2, body);
}
+ private static TPipeTransferReq replayableRequest(final int value) {
+ return request((byte) 1, (short) 10, new byte[] {(byte) value});
+ }
+
private static TPipeTransferReq request(final byte version, final short type, final byte[] body) {
return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body));
}
From 4451babdf117666b14245a04ea1561b1fdafbe0c Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Mon, 21 Sep 2026 16:43:51 +0800
Subject: [PATCH 4/8] Add managed logical backup export workflow
---
.../apache/iotdb/cli/i18n/CliMessages.java | 21 +-
.../apache/iotdb/cli/i18n/CliMessages.java | 21 +-
.../tool/pipe/PipeLogicalBackupTool.java | 228 ++++++++++++++++--
3 files changed, 250 insertions(+), 20 deletions(-)
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 8aa2feccc907..064dd16db8b0 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -128,8 +128,8 @@ private CliMessages() {}
public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
"Logical backup request type %d failed: %s";
public static final String
- LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
- "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB =
+ "Use --input to specify the input. Export also requires --output; export can use --backup-dir to create, monitor, and clean up a snapshot Pipe automatically. Import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
"pipe-logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
@@ -151,5 +151,22 @@ private CliMessages() {}
"Logical backup checkpoint is invalid: %s";
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
"Unsupported logical backup export format: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 =
+ "Logical backup export timeout and poll interval must be positive";
+ public static final String
+ LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B =
+ "Logical backup export %s: state=%s, remaining=%s, estimated_seconds=%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 =
+ "Logical backup export pipe %s failed: %s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B =
+ "Logical backup export pipe %s timed out after %d seconds";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE =
+ "Logical backup option --%s must be an integer";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 =
+ "Logical backup option --%s has invalid identifier: %s";
}
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index 35346133f19e..d08e2053fa8c 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -124,8 +124,8 @@ private CliMessages() {}
public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 =
"逻辑备份请求类型 %d 执行失败：%s";
public static final String
- LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E =
- "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
+ LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB =
+ "使用 --input 指定输入。导出还需要 --output；导出也可使用 --backup-dir 自动创建、监控并清理快照 Pipe。导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
"pipe-logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
@@ -148,4 +148,21 @@ private CliMessages() {}
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 =
"不支持的逻辑备份导出格式：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 =
+ "逻辑备份导出 timeout 和轮询间隔必须为正数";
+ public static final String
+ LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B =
+ "逻辑备份导出 %s：状态=%s，剩余=%s，预计秒数=%s";
+ public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 =
+ "逻辑备份导出 pipe %s 失败：%s";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B =
+ "逻辑备份导出 pipe %s 在 %d 秒后超时";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE =
+ "逻辑备份选项 --%s 必须是整数";
+ public static final String
+ EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 =
+ "逻辑备份选项 --%s 的标识符无效：%s";
}
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
index 1d2315f17c04..218f36709317 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -30,9 +30,11 @@
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant;
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req;
+import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
+import org.apache.iotdb.session.Session;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -97,6 +99,11 @@ public final class PipeLogicalBackupTool {
private static final String OPTION_DRY_RUN = "dry-run";
private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete";
private static final String OPTION_JSON = "json";
+ private static final String OPTION_BACKUP_DIR = "backup-dir";
+ private static final String OPTION_PIPE_NAME = "pipe-name";
+ private static final String OPTION_BACKUP_ID = "backup-id";
+ private static final String OPTION_TIMEOUT_SECONDS = "timeout-seconds";
+ private static final String OPTION_POLL_INTERVAL_SECONDS = "poll-interval-seconds";
private static final int MAX_ARCHIVE_ENTRIES = 1_000_000;
private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024;
private static final String CHECKPOINT_FORMAT_NAME =
@@ -162,9 +169,15 @@ private static CommandLine parse(final String command, final String[] args)
addOption(options, OPTION_DRY_RUN, false, false);
addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
addOption(options, OPTION_JSON, false, false);
+ addOption(options, OPTION_BACKUP_DIR, true, false);
+ addOption(options, OPTION_PIPE_NAME, true, false);
+ addOption(options, OPTION_BACKUP_ID, true, false);
+ addOption(options, OPTION_TIMEOUT_SECONDS, true, false);
+ addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false);
final CommandLine line = new DefaultParser().parse(options, args);
final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
- if (input == null || input.isBlank()) {
+ if ((input == null || input.isBlank())
+ && (!COMMAND_EXPORT.equals(command) || !line.hasOption(OPTION_BACKUP_DIR))) {
throw new ParseException(
String.format(
CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
@@ -231,32 +244,108 @@ private static int verify(final CommandLine line) throws IOException {
return 0;
}
- private static int export(final CommandLine line) throws IOException {
+ private static int export(final CommandLine line) throws Exception {
+ if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) {
+ return managedExport(line);
+ }
final Path source = Path.of(input(line)).toAbsolutePath().normalize();
final Path target =
Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
final List streams = read(line);
+ return exportArchive(source, target, streams);
+ }
+
+ private static int managedExport(final CommandLine line) throws Exception {
+ final String host = required(line, OPTION_HOST);
+ final int port = Integer.parseInt(required(line, OPTION_PORT));
+ final String user = line.getOptionValue(OPTION_USER, "root");
+ final String password = password(line);
+ final Path backupRoot = Path.of(required(line, OPTION_BACKUP_DIR)).toAbsolutePath().normalize();
+ final Path target =
+ Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
+ final String pipeName =
+ line.getOptionValue(
+ OPTION_PIPE_NAME,
+ "logical_backup_export_" + UUID.randomUUID().toString().replace('-', '_'));
+ validateIdentifier(pipeName, OPTION_PIPE_NAME);
+ final String backupId = line.getOptionValue(OPTION_BACKUP_ID, pipeName);
+ validateIdentifier(backupId, OPTION_BACKUP_ID);
+ final long timeoutSeconds = longOption(line, OPTION_TIMEOUT_SECONDS, 3600L);
+ final long pollIntervalSeconds = longOption(line, OPTION_POLL_INTERVAL_SECONDS, 2L);
+ if (timeoutSeconds <= 0 || pollIntervalSeconds <= 0) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769);
+ }
+
+ final String createSql =
+ "CREATE PIPE "
+ + pipeName
+ + " WITH SOURCE ('source'='iotdb-source','source.mode'='snapshot','source.inclusion'='all',"
+ + "'source.capture.tree'='true','source.capture.table'='true',"
+ + "'source.realtime.enable'='false') WITH SINK ('sink'='logical-backup-sink','sink.dir'='"
+ + sqlString(backupRoot.toString())
+ + "','sink.backup-id'='"
+ + sqlString(backupId)
+ + "','sink.resume'='append')";
+ boolean created = false;
+ try (final Session session =
+ new Session.Builder().host(host).port(port).username(user).password(password).build()) {
+ session.open();
+ session.executeNonQueryStatement(createSql);
+ created = true;
+ session.executeNonQueryStatement("START PIPE " + pipeName);
+ waitForSnapshotPipe(session, pipeName, timeoutSeconds, pollIntervalSeconds);
+ final Path source = backupRoot.resolve(backupId).normalize();
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ return exportArchive(source, target, streams);
+ } finally {
+ if (created) {
+ try (final Session cleanup =
+ new Session.Builder().host(host).port(port).username(user).password(password).build()) {
+ cleanup.open();
+ try {
+ cleanup.executeNonQueryStatement("STOP PIPE " + pipeName);
+ } catch (final Exception ignored) {
+ // A completed snapshot pipe may already have been auto-dropped.
+ }
+ try {
+ cleanup.executeNonQueryStatement("DROP PIPE " + pipeName);
+ } catch (final Exception ignored) {
+ // Keep the original export failure when cleanup races with auto-drop.
+ }
+ } catch (final Exception ignored) {
+ // The archive is complete once the generated pipe disappears; cleanup is best effort.
+ }
+ }
+ }
+ }
+
+ private static int exportArchive(
+ final Path source, final Path target, final List streams) throws IOException {
+ final Path normalizedSource = source.toAbsolutePath().normalize();
+ final Path normalizedTarget = target.toAbsolutePath().normalize();
final Path root;
- if (Files.isDirectory(source)) {
- root = source;
- } else if (Files.isRegularFile(source)
- && "manifest.json".equals(source.getFileName().toString())) {
- root = source.getParent();
+ if (Files.isDirectory(normalizedSource)) {
+ root = normalizedSource;
+ } else if (Files.isRegularFile(normalizedSource)
+ && "manifest.json".equals(normalizedSource.getFileName().toString())) {
+ root = normalizedSource.getParent();
} else {
throw new IOException(
String.format(
CliMessages
.EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
- source));
+ normalizedSource));
}
- if (target.startsWith(root)) {
+ if (normalizedTarget.startsWith(root)) {
throw new IOException(
CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994);
}
- if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
- throw new java.nio.file.FileAlreadyExistsException(target.toString());
+ if (Files.exists(normalizedTarget, LinkOption.NOFOLLOW_LINKS)) {
+ throw new java.nio.file.FileAlreadyExistsException(normalizedTarget.toString());
}
- final Path parent = target.toAbsolutePath().normalize().getParent();
+ final Path parent = normalizedTarget.getParent();
if (parent != null) {
Files.createDirectories(parent);
}
@@ -276,9 +365,9 @@ private static int export(final CommandLine line) throws IOException {
archiveChannel.force(true);
}
try {
- Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE);
+ Files.move(temporary, normalizedTarget, StandardCopyOption.ATOMIC_MOVE);
} catch (final java.nio.file.AtomicMoveNotSupportedException e) {
- Files.move(temporary, target);
+ Files.move(temporary, normalizedTarget);
}
published = true;
} finally {
@@ -288,10 +377,112 @@ private static int export(final CommandLine line) throws IOException {
}
System.out.println(
String.format(
- CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target));
+ CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280,
+ normalizedSource,
+ normalizedTarget));
return 0;
}
+ private static void waitForSnapshotPipe(
+ final Session session,
+ final String pipeName,
+ final long timeoutSeconds,
+ final long pollSeconds)
+ throws Exception {
+ final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L;
+ while (System.nanoTime() < deadline) {
+ final PipeStatus status = readPipeStatus(session, pipeName);
+ if (status == null) {
+ return;
+ } else {
+ System.out.println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B,
+ pipeName,
+ status.state,
+ status.remainingEvents,
+ status.estimatedSeconds));
+ if (status.state != null
+ && (status.state.equalsIgnoreCase("ERROR")
+ || status.state.equalsIgnoreCase("STOPPED"))) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190,
+ pipeName,
+ status.exceptionMessage));
+ }
+ }
+ Thread.sleep(pollSeconds * 1000L);
+ }
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B,
+ pipeName,
+ timeoutSeconds));
+ }
+
+ private static PipeStatus readPipeStatus(final Session session, final String pipeName)
+ throws Exception {
+ try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW PIPE " + pipeName)) {
+ final SessionDataSet.DataIterator iterator = dataSet.iterator();
+ if (!iterator.next()) {
+ return null;
+ }
+ final PipeStatus status = new PipeStatus();
+ status.state = value(iterator, dataSet, "State");
+ status.remainingEvents = value(iterator, dataSet, "RemainingEventCount");
+ status.estimatedSeconds = value(iterator, dataSet, "EstimatedRemainingSeconds");
+ status.exceptionMessage = value(iterator, dataSet, "ExceptionMessage");
+ return status;
+ }
+ }
+
+ private static String value(
+ final SessionDataSet.DataIterator iterator, final SessionDataSet dataSet, final String name)
+ throws Exception {
+ for (final String column : dataSet.getColumnNames()) {
+ if (column.equalsIgnoreCase(name)) {
+ return iterator.getString(column);
+ }
+ }
+ return "Unknown";
+ }
+
+ private static long longOption(final CommandLine line, final String name, final long fallback)
+ throws ParseException {
+ try {
+ return Long.parseLong(line.getOptionValue(name, Long.toString(fallback)));
+ } catch (final NumberFormatException e) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE, name));
+ }
+ }
+
+ private static void validateIdentifier(final String value, final String option)
+ throws IOException {
+ if (!value.matches("[A-Za-z_][A-Za-z0-9_]{0,127}")) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204,
+ option,
+ value));
+ }
+ }
+
+ private static String sqlString(final String value) {
+ return value.replace("'", "''");
+ }
+
+ private static final class PipeStatus {
+ private String state;
+ private String remainingEvents;
+ private String estimatedSeconds;
+ private String exceptionMessage;
+ }
+
private static List exportFiles(final Path root, final List streams)
throws IOException {
final Set files = new LinkedHashSet<>();
@@ -748,7 +939,7 @@ private static void printUsage() {
120,
CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE,
CliMessages
- .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E,
+ .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB,
optionsForHelp(),
2,
2,
@@ -772,6 +963,11 @@ private static Options optionsForHelp() {
addOption(options, OPTION_DRY_RUN, false, false);
addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
addOption(options, OPTION_JSON, false, false);
+ addOption(options, OPTION_BACKUP_DIR, true, false);
+ addOption(options, OPTION_PIPE_NAME, true, false);
+ addOption(options, OPTION_BACKUP_ID, true, false);
+ addOption(options, OPTION_TIMEOUT_SECONDS, true, false);
+ addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false);
return options;
}
From d38a79627b7c859eb3fcbae1f394aeb88443776b Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Tue, 22 Sep 2026 16:01:26 +0800
Subject: [PATCH 5/8] Rename logical backup user-facing scripts
---
.../src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java | 2 +-
.../src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java | 2 +-
.../{export-pipe-logical-backup.sh => export-logical-backup.sh} | 2 +-
.../{import-pipe-logical-backup.sh => import-logical-backup.sh} | 2 +-
scripts/tools/{pipe-logical-backup.sh => logical-backup.sh} | 0
...export-pipe-logical-backup.bat => export-logical-backup.bat} | 2 +-
...import-pipe-logical-backup.bat => import-logical-backup.bat} | 2 +-
.../windows/{pipe-logical-backup.bat => logical-backup.bat} | 0
8 files changed, 6 insertions(+), 6 deletions(-)
rename scripts/tools/{export-pipe-logical-backup.sh => export-logical-backup.sh} (93%)
rename scripts/tools/{import-pipe-logical-backup.sh => import-logical-backup.sh} (93%)
rename scripts/tools/{pipe-logical-backup.sh => logical-backup.sh} (100%)
rename scripts/tools/windows/{export-pipe-logical-backup.bat => export-logical-backup.bat} (94%)
rename scripts/tools/windows/{import-pipe-logical-backup.bat => import-logical-backup.bat} (94%)
rename scripts/tools/windows/{pipe-logical-backup.bat => logical-backup.bat} (100%)
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 064dd16db8b0..1316415682f5 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -131,7 +131,7 @@ private CliMessages() {}
LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB =
"Use --input to specify the input. Export also requires --output; export can use --backup-dir to create, monitor, and clean up a snapshot Pipe automatically. Import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords.";
public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
- "pipe-logical-backup ";
+ "logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
"Logical backup archive entry is unsafe: %s";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index d08e2053fa8c..c26071d0bf5f 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -127,7 +127,7 @@ private CliMessages() {}
LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB =
"使用 --input 指定输入。导出还需要 --output；导出也可使用 --backup-dir 自动创建、监控并清理快照 Pipe。导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。";
public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE =
- "pipe-logical-backup ";
+ "logical-backup ";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 =
"逻辑备份归档项路径不安全：%s";
public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 =
diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-logical-backup.sh
similarity index 93%
rename from scripts/tools/export-pipe-logical-backup.sh
rename to scripts/tools/export-logical-backup.sh
index 8a0a4f8a1834..fcc072bb5c1e 100644
--- a/scripts/tools/export-pipe-logical-backup.sh
+++ b/scripts/tools/export-logical-backup.sh
@@ -18,4 +18,4 @@
# under the License.
#
-exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@"
+exec "$(dirname "$0")/logical-backup.sh" export "$@"
diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-logical-backup.sh
similarity index 93%
rename from scripts/tools/import-pipe-logical-backup.sh
rename to scripts/tools/import-logical-backup.sh
index 6e4e069210de..ebd38227a289 100644
--- a/scripts/tools/import-pipe-logical-backup.sh
+++ b/scripts/tools/import-logical-backup.sh
@@ -18,4 +18,4 @@
# under the License.
#
-exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@"
+exec "$(dirname "$0")/logical-backup.sh" import "$@"
diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/logical-backup.sh
similarity index 100%
rename from scripts/tools/pipe-logical-backup.sh
rename to scripts/tools/logical-backup.sh
diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-logical-backup.bat
similarity index 94%
rename from scripts/tools/windows/export-pipe-logical-backup.bat
rename to scripts/tools/windows/export-logical-backup.bat
index 42e1df8afb27..d79ce5a43d98 100644
--- a/scripts/tools/windows/export-pipe-logical-backup.bat
+++ b/scripts/tools/windows/export-logical-backup.bat
@@ -17,5 +17,5 @@
@REM under the License.
@REM
-@call "%~dp0pipe-logical-backup.bat" export %*
+@call "%~dp0logical-backup.bat" export %*
@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-logical-backup.bat
similarity index 94%
rename from scripts/tools/windows/import-pipe-logical-backup.bat
rename to scripts/tools/windows/import-logical-backup.bat
index 0fba332d5a2e..bfe85ba3943f 100644
--- a/scripts/tools/windows/import-pipe-logical-backup.bat
+++ b/scripts/tools/windows/import-logical-backup.bat
@@ -17,5 +17,5 @@
@REM under the License.
@REM
-@call "%~dp0pipe-logical-backup.bat" import %*
+@call "%~dp0logical-backup.bat" import %*
@exit /B %ERRORLEVEL%
diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/logical-backup.bat
similarity index 100%
rename from scripts/tools/windows/pipe-logical-backup.bat
rename to scripts/tools/windows/logical-backup.bat
From bc0a6e2d6249ec4ae9afb0232b4cb7fd5865ab4f Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Thu, 24 Sep 2026 20:19:59 +0800
Subject: [PATCH 6/8] Add ConfigNode logical backup support
---
.../tool/pipe/PipeLogicalBackupTool.java | 42 ++++
.../tool/pipe/PipeLogicalBackupToolTest.java | 39 +++-
.../PipeConfigRegionSinkConstructor.java | 3 +
.../logicalbackup/LogicalBackupSink.java | 183 ++++++++++++++++++
.../PipeConfigRegionSinkConstructorTest.java | 49 +++++
.../sink/PipeConfigNodeThriftRequestTest.java | 7 +
.../iotdb/db/i18n/DataNodePipeMessages.java | 4 +-
.../iotdb/db/i18n/DataNodePipeMessages.java | 4 +-
.../logicalbackup/LogicalBackupSink.java | 32 +--
.../logicalbackup/LogicalBackupSinkTest.java | 14 ++
.../plugin/builtin/BuiltinPipePlugin.java | 2 +-
.../LogicalBackupArchiveReader.java | 18 +-
.../request/PipeTransferHandshakeV2Req.java | 29 ++-
.../LogicalBackupArchiveReaderTest.java | 43 +++-
14 files changed, 430 insertions(+), 39 deletions(-)
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
index 218f36709317..631353027a61 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -29,6 +29,8 @@
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -536,6 +538,10 @@ private static int importBackup(final CommandLine line) throws Exception {
"",
"")) {
handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
+ if (containsConfigStream(streams)) {
+ configNodeHandshake(
+ client, streams.get(0).getManifest().timestampPrecision, user, password);
+ }
long importedGroups = 0;
for (final BackupStream stream : streams) {
importedGroups +=
@@ -557,6 +563,11 @@ private static int importBackup(final CommandLine line) throws Exception {
return 0;
}
+ static boolean containsConfigStream(final List streams) {
+ return streams.stream()
+ .anyMatch(stream -> "config".equalsIgnoreCase(stream.getManifest().streamType));
+ }
+
static long importStream(
final Path checkpoint,
final ImportCheckpoint checkpointState,
@@ -695,6 +706,37 @@ private static void handshake(
}
}
+ private static void configNodeHandshake(
+ final IoTDBSyncClient client,
+ final String timestampPrecision,
+ final String user,
+ final String password)
+ throws IOException, TException {
+ final Map params = new HashMap<>();
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId());
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password);
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true");
+ params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false");
+ final TPipeTransferResp response =
+ client.pipeTransfer(
+ PipeTransferHandshakeV2Req.toTPipeTransferReq(
+ PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params));
+ if (response == null
+ || response.getStatus() == null
+ || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697,
+ response == null ? "null" : response.getStatus()));
+ }
+ }
+
static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
throws TException, IOException {
final TPipeTransferResp response = client.pipeTransfer(request);
diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
index 6e7af8726ab3..f39b87f0754e 100644
--- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
+++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
@@ -116,6 +116,22 @@ public void testCheckpointSourceIdentityIncludesContent() throws Exception {
Assert.assertNotEquals(before, after);
}
+ @Test
+ public void testConfigStreamDetection() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path data = root.resolve("data");
+ final Path config = root.resolve("config");
+ writeBackup(data);
+ writeBackup(config, "config-0", "config", Collections.singletonList(request((short) 200)));
+
+ Assert.assertFalse(
+ PipeLogicalBackupTool.containsConfigStream(
+ new LogicalBackupArchiveReader().read(data, false)));
+ Assert.assertTrue(
+ PipeLogicalBackupTool.containsConfigStream(
+ new LogicalBackupArchiveReader().read(config, false)));
+ }
+
@Test
public void testZipSlipIsRejected() throws Exception {
final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip");
@@ -236,12 +252,21 @@ private static void writeBackup(final Path directory) throws Exception {
private static void writeBackup(final Path directory, final List requests)
throws Exception {
+ writeBackup(directory, "data-1", "data", requests);
+ }
+
+ private static void writeBackup(
+ final Path directory,
+ final String streamId,
+ final String streamType,
+ final List requests)
+ throws Exception {
final LogicalBackupManifest manifest = new LogicalBackupManifest();
manifest.backupId = "backup";
manifest.pipeName = "pipe";
manifest.pipeCreationTime = 1;
- manifest.streamId = "data-1";
- manifest.streamType = "data";
+ manifest.streamId = streamId;
+ manifest.streamType = streamType;
manifest.timestampPrecision = "ms";
try (final LogicalBackupWriter writer =
new LogicalBackupWriter(
@@ -251,9 +276,17 @@ private static void writeBackup(final Path directory, final List requests = new ArrayList<>();
+ appendFile(requests, e.getSnapshotFile());
+ appendFile(requests, e.getTemplateFile());
+ final File snapshot = e.getSnapshotFile();
+ final File template = e.getTemplateFile();
+ requests.add(
+ PipeTransferConfigSnapshotSealReq.toTPipeTransferReq(
+ e.getTreePattern().getPattern(),
+ e.getTablePattern().getDatabasePattern(),
+ e.getTablePattern().getTablePattern(),
+ e.getTreePattern().isTreeModelDataAllowedToBeCaptured(),
+ e.getTablePattern().isTableModelDataAllowedToBeCaptured(),
+ snapshot.getName(),
+ snapshot.length(),
+ template == null ? null : template.getName(),
+ template == null ? 0 : template.length(),
+ e.getFileType(),
+ e.toSealTypeString(),
+ e.getAuthUserName()));
+ writer.writeEvent(
+ UUID.randomUUID(), System.currentTimeMillis(), requests, "config-snapshot");
+ } finally {
+ e.decreaseReferenceCount(getClass().getName(), false);
+ }
+ }
+ }
+ }
+
+ private static void appendFile(final List requests, final File file)
+ throws Exception {
+ if (file == null) return;
+ try (FileInputStream input = new FileInputStream(file)) {
+ final byte[] buffer = new byte[FILE_PIECE_BYTES];
+ long offset = 0;
+ int length;
+ while ((length = input.read(buffer)) > 0) {
+ requests.add(
+ PipeTransferConfigSnapshotPieceReq.toTPipeTransferReq(
+ file.getName(), offset, Arrays.copyOf(buffer, length)));
+ offset += length;
+ }
+ }
+ }
+
+ @Override
+ public void handshake() {}
+
+ @Override
+ public void heartbeat() throws Exception {
+ if (writer != null) writer.heartbeat();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java
new file mode 100644
index 000000000000..3031e2c6b216
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.pipe.agent.plugin;
+
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup.LogicalBackupSink;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+
+public class PipeConfigRegionSinkConstructorTest {
+
+ @Test
+ public void testLogicalBackupAliasesUseConfigRegionSink() {
+ final PipeConfigRegionSinkConstructor constructor = new PipeConfigRegionSinkConstructor();
+
+ Assert.assertTrue(
+ constructor.reflectPlugin(
+ new PipeParameters(
+ Collections.singletonMap(
+ PipeSinkConstant.CONNECTOR_KEY, "logical-backup-connector")))
+ instanceof LogicalBackupSink);
+ Assert.assertTrue(
+ constructor.reflectPlugin(
+ new PipeParameters(
+ Collections.singletonMap(PipeSinkConstant.SINK_KEY, "logical-backup-sink")))
+ instanceof LogicalBackupSink);
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java
index bd77d72d1bef..e8a17e5d524d 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java
@@ -23,6 +23,7 @@
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan;
import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigNodeHandshakeV1Req;
@@ -93,12 +94,18 @@ public void testPipeTransferConfigHandshakeV2Req() throws IOException {
final PipeTransferConfigNodeHandshakeV2Req req =
PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferReq(params);
+ final TPipeTransferReq commonReq =
+ PipeTransferHandshakeV2Req.toTPipeTransferReq(
+ PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params);
final int originalBodyPosition = req.body.position();
final PipeTransferConfigNodeHandshakeV2Req deserializeReq =
PipeTransferConfigNodeHandshakeV2Req.fromTPipeTransferReq(req);
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
+ Assert.assertEquals(req.getVersion(), commonReq.getVersion());
+ Assert.assertEquals(req.getType(), commonReq.getType());
+ Assert.assertArrayEquals(req.getBody(), commonReq.getBody());
Assert.assertEquals(originalBodyPosition, req.body.position());
Assert.assertEquals(params, deserializeReq.getParams());
}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 8c6200718838..5744a1bd34cf 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2654,8 +2654,8 @@ private DataNodePipeMessages() {}
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
"Unsupported logical backup resume policy: %s";
public static final String
- EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
- "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory";
+ EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B =
+ "ConfigRegion and DataRegion logical backup require explicit sink.resume=append (or connector.resume=append) and a shared backup directory";
public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
"Unsupported Pipe event for logical backup: %s";
public static final String
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 44f7c0269199..d5200ee93dfa 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2478,8 +2478,8 @@ private DataNodePipeMessages() {}
public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 =
"不支持的逻辑备份恢复策略：%s";
public static final String
- EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 =
- "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录";
+ EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B =
+ "ConfigRegion 和 DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录";
public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 =
"逻辑备份不支持 Pipe 事件：%s";
public static final String
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
index ef152b58b7c3..c48238a5ce22 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -75,12 +75,12 @@
public class LogicalBackupSink implements PipeConnector {
private static final int FILE_PIECE_BYTES = 64 * 1024;
- private LogicalBackupWriter writer;
+ protected LogicalBackupWriter writer;
private String pipeName;
private long creationTime;
private int regionId;
- private boolean includeHeartbeat;
- private String unsupportedEventPolicy;
+ protected boolean includeHeartbeat;
+ protected String unsupportedEventPolicy;
private final String streamType;
public LogicalBackupSink() {
@@ -160,7 +160,7 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH,
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
- validateDataRegionResumePolicy(parameters);
+ validateSharedDirectoryResumePolicy(parameters);
final String policy =
parameters
.getStringOrDefault(
@@ -188,9 +188,9 @@ private static void validateSynonymAttributes(
false);
}
- private void validateDataRegionResumePolicy(final PipeParameters parameters)
+ private void validateSharedDirectoryResumePolicy(final PipeParameters parameters)
throws PipeParameterNotValidException {
- if (!"data".equals(streamType)) {
+ if (!"data".equals(streamType) && !"config".equals(streamType)) {
return;
}
final boolean explicitlyConfigured =
@@ -207,7 +207,7 @@ private void validateDataRegionResumePolicy(final PipeParameters parameters)
|| !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) {
throw new PipeParameterNotValidException(
DataNodePipeMessages
- .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8);
+ .EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B);
}
}
@@ -277,7 +277,7 @@ public void customize(
PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE)
.toLowerCase(Locale.ROOT);
- validateDataRegionResumePolicy(parameters);
+ validateSharedDirectoryResumePolicy(parameters);
final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume);
if (!append
&& !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)
@@ -349,7 +349,7 @@ public void customize(
manifest.backupId = backupId;
manifest.pipeName = pipeName;
manifest.pipeCreationTime = creationTime;
- manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId();
+ manifest.sourceClusterId = getSourceClusterId();
manifest.sourceVersion = IoTDBConstant.VERSION;
manifest.timestampPrecision =
CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
@@ -369,6 +369,10 @@ public void customize(
append);
}
+ protected String getSourceClusterId() {
+ return IoTDBDescriptor.getInstance().getConfig().getClusterId();
+ }
+
@Override
public void handshake() {}
@@ -542,7 +546,7 @@ private List requests(final PipeInsertNodeTabletInsertionEvent
return requests;
}
- private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier)
+ protected void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier)
throws Exception {
if (!event.increaseReferenceCount(getClass().getName())) {
return;
@@ -556,11 +560,11 @@ private void transferEnriched(final EnrichedEvent event, final RequestSupplier r
}
@FunctionalInterface
- private interface RequestSupplier {
+ protected interface RequestSupplier {
List get() throws Exception;
}
- private void handleUnsupported(final Event event) throws Exception {
+ protected void handleUnsupported(final Event event) throws Exception {
if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) {
writer.recordSkippedEvent(
System.currentTimeMillis(), event == null ? "null" : event.getClass().getName());
@@ -572,7 +576,7 @@ private void handleUnsupported(final Event event) throws Exception {
event == null ? "null" : event.getClass().getName()));
}
- private UUID eventId(final EnrichedEvent event) {
+ protected UUID eventId(final EnrichedEvent event) {
return UUID.nameUUIDFromBytes(
(event.getClass().getName()
+ '\0'
@@ -594,7 +598,7 @@ private String commitIds(final EnrichedEvent event) {
}
}
- private String metadata(final Event event) {
+ protected String metadata(final Event event) {
if (event instanceof EnrichedEvent) {
final EnrichedEvent enriched = (EnrichedEvent) event;
return "pipe="
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
index 37c3d622b4f5..2354e8309ab9 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
@@ -67,6 +67,20 @@ public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception {
.validate(new PipeParameterValidator(new PipeParameters(attributes)));
}
+ @Test
+ public void testConfigRegionRequiresExplicitAppendResumePolicy() throws Exception {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup");
+
+ assertInvalid(new LogicalBackupSink("config"), attributes);
+
+ attributes.put(
+ PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY,
+ PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND);
+ new LogicalBackupSink("config")
+ .validate(new PipeParameterValidator(new PipeParameters(attributes)));
+ }
+
private static void assertInvalid(
final LogicalBackupSink sink, final Map attributes) throws Exception {
try {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index 11859bb6fe61..d9ff67f160e8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -38,7 +38,6 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
@@ -224,6 +223,7 @@ public String getClassName() {
WEBSOCKET_SINK.getPipePluginName().toUpperCase(),
OPC_UA_SINK.getPipePluginName().toUpperCase(),
OPC_DA_SINK.getPipePluginName().toUpperCase(),
+ LOGICAL_BACKUP_SINK.getPipePluginName().toUpperCase(),
SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(),
IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase())));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
index 6ac9ca23f67b..b378a16136c1 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
@@ -52,7 +52,10 @@ public class LogicalBackupArchiveReader {
PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(),
PipeRequestType.TRANSFER_PLAN_NODE.getType(),
PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(),
- PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType());
+ PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(),
+ PipeRequestType.TRANSFER_CONFIG_PLAN.getType(),
+ PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType(),
+ PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType());
public List read(final Path source, final boolean allowIncomplete)
throws IOException {
@@ -84,12 +87,21 @@ public List read(final Path source, final boolean allowIncomplete)
}
streams.sort(
Comparator.comparingInt(
- (BackupStream stream) ->
- "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1)
+ (BackupStream stream) -> streamOrder(stream.getManifest().streamType))
.thenComparing((BackupStream stream) -> stream.getManifest().streamId));
return Collections.unmodifiableList(streams);
}
+ private static int streamOrder(final String streamType) {
+ if ("config".equalsIgnoreCase(streamType)) {
+ return 0;
+ }
+ if ("schema".equalsIgnoreCase(streamType)) {
+ return 1;
+ }
+ return 2;
+ }
+
private static void validateArchiveIdentity(
final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException {
validateArchiveIdentityField("backupId", expected.backupId, actual.backupId);
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
index 472679494729..d7e45cad5ae7 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java
@@ -41,6 +41,20 @@ public Map getParams() {
protected abstract PipeRequestType getPlanType();
+ public static TPipeTransferReq toTPipeTransferReq(
+ final PipeRequestType requestType, final Map params) throws IOException {
+ final TPipeTransferReq request = new TPipeTransferReq();
+ request.version = IoTDBSinkRequestVersion.VERSION_1.getVersion();
+ request.type = requestType.getType();
+ try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
+ writeParams(params, outputStream);
+ request.body =
+ ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ }
+ return request;
+ }
+
/////////////////////////////// Thrift ///////////////////////////////
protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params)
@@ -49,11 +63,7 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map entry : params.entrySet()) {
- ReadWriteIOUtils.write(entry.getKey(), outputStream);
- ReadWriteIOUtils.write(entry.getValue(), outputStream);
- }
+ writeParams(params, outputStream);
this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
@@ -61,6 +71,15 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params, final DataOutputStream outputStream) throws IOException {
+ ReadWriteIOUtils.write(params.size(), outputStream);
+ for (final Map.Entry entry : params.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ }
+
protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq(
TPipeTransferReq transferReq) {
Map params = new HashMap<>();
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
index 05f6298e95fa..10db588f2c3c 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
@@ -35,6 +35,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -47,16 +48,18 @@ public class LogicalBackupArchiveReaderTest {
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
- public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception {
+ public void testReadCompleteArchiveAndOrderConfigThenSchemaThenData() throws Exception {
final Path root = temporaryFolder.newFolder().toPath();
writeStream(root.resolve("data"), "data-stream", "data", 4096);
writeStream(root.resolve("schema"), "schema-stream", "schema", 4096);
+ writeStream(root.resolve("config"), "config-stream", "config", 4096);
final List streams = new LogicalBackupArchiveReader().read(root, false);
- Assert.assertEquals(2, streams.size());
- Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId);
- Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId);
+ Assert.assertEquals(3, streams.size());
+ Assert.assertEquals("config-stream", streams.get(0).getManifest().streamId);
+ Assert.assertEquals("schema-stream", streams.get(1).getManifest().streamId);
+ Assert.assertEquals("data-stream", streams.get(2).getManifest().streamId);
Assert.assertEquals(1, streams.get(0).getEventGroups().size());
Assert.assertEquals(4, streams.get(0).getRecords().size());
Assert.assertEquals(
@@ -175,7 +178,7 @@ public void testNonReplayableRequestTypeIsRejected() throws Exception {
Collections.singletonList(
new TPipeTransferReq()
.setVersion((byte) 1)
- .setType((short) 200)
+ .setType(Short.MAX_VALUE)
.setBody(ByteBuffer.wrap(new byte[] {1}))),
"metadata");
}
@@ -183,6 +186,23 @@ public void testNonReplayableRequestTypeIsRejected() throws Exception {
assertReadFails(directory, false);
}
+ @Test
+ public void testConfigRequestTypesAreReplayable() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ try (final LogicalBackupWriter writer =
+ writer(directory, manifest("config-0", "config"), 4096)) {
+ writer.writeEvent(
+ UUID.randomUUID(),
+ 1,
+ Arrays.asList(request((short) 200), request((short) 201), request((short) 202)),
+ "metadata");
+ }
+
+ final List streams = new LogicalBackupArchiveReader().read(directory, false);
+ Assert.assertEquals(1, streams.size());
+ Assert.assertEquals(3, streams.get(0).getEventGroups().get(0).getRequests().size());
+ }
+
@Test
public void testDuplicateEventGroupIdIsRejected() throws Exception {
final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
@@ -239,10 +259,15 @@ private static TPipeTransferReq request(final int value) {
}
private static TPipeTransferReq request(final byte[] body) {
- return new TPipeTransferReq()
- .setVersion((byte) 1)
- .setType((short) 10)
- .setBody(ByteBuffer.wrap(body));
+ return request((short) 10, body);
+ }
+
+ private static TPipeTransferReq request(final short type) {
+ return request(type, new byte[] {1});
+ }
+
+ private static TPipeTransferReq request(final short type, final byte[] body) {
+ return new TPipeTransferReq().setVersion((byte) 1).setType(type).setBody(ByteBuffer.wrap(body));
}
private static Path onlySegment(final Path directory) throws IOException {
From 7311c1454f2cd6d46a0b92c35c7566d7a130258d Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Fri, 9 Oct 2026 12:10:42 +0800
Subject: [PATCH 7/8] Fix logical backup builtin plugin import
---
.../commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index d9ff67f160e8..c06b8cf45869 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -38,6 +38,7 @@
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
+import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
From 6355496a6f82ae4bbd25b34f29eabc5996f2ca30 Mon Sep 17 00:00:00 2001
From: Caideyipi <87789683+Caideyipi@users.noreply.github.com>
Date: Mon, 19 Oct 2026 16:40:57 +0800
Subject: [PATCH 8/8] feat: add logical backup dump and restore workflow
---
.../manual/IoTDBPipeLogicalBackupIT.java | 298 +++++
iotdb-client/cli/LOGICAL_BACKUP.md | 138 +++
.../apache/iotdb/cli/i18n/CliMessages.java | 40 +
.../apache/iotdb/cli/i18n/CliMessages.java | 40 +
.../tool/pipe/LogicalBackupDumpOptions.java | 275 +++++
.../tool/pipe/PipeLogicalBackupTool.java | 1059 +++++++++++++----
.../tool/pipe/PipeLogicalBackupToolTest.java | 258 +++-
.../heartbeat/PipeHeartbeatParser.java | 67 +-
.../logicalbackup/LogicalBackupSink.java | 97 +-
.../heartbeat/PipeHeartbeatParserTest.java | 32 +
.../subtask/sink/PipeSinkSubtaskManager.java | 32 +-
.../logicalbackup/LogicalBackupSink.java | 227 ++--
.../dataregion/IoTDBDataRegionSource.java | 11 +
.../PipeRealtimeDataRegionSource.java | 25 +
.../sink/PipeSinkSubtaskManagerTest.java | 21 +
.../logicalbackup/LogicalBackupSinkTest.java | 105 ++
.../PipeRealtimeDataRegionSourceTest.java | 20 +
.../commons/i18n/LogicalBackupMessages.java | 10 +
.../commons/i18n/LogicalBackupMessages.java | 10 +
.../LogicalBackupArchiveReader.java | 147 ++-
.../logicalbackup/LogicalBackupCatalog.java | 277 +++++
.../logicalbackup/LogicalBackupManifest.java | 4 +
.../logicalbackup/LogicalBackupSettings.java | 199 ++++
.../logicalbackup/LogicalBackupWriter.java | 21 +-
.../pipe/source/IoTDBNonDataRegionSource.java | 51 +-
.../LogicalBackupArchiveReaderTest.java | 105 +-
.../LogicalBackupSettingsTest.java | 99 ++
.../LogicalBackupWriterTest.java | 11 +-
.../source/IoTDBNonDataRegionSourceTest.java | 60 +-
29 files changed, 3260 insertions(+), 479 deletions(-)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java
create mode 100644 iotdb-client/cli/LOGICAL_BACKUP.md
create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java
create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java
new file mode 100644
index 000000000000..4629bcb88750
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.pipe.it.dual.treemodel.manual;
+
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.tool.pipe.PipeLogicalBackupTool;
+
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.TimeUnit;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({MultiClusterIT2DualTreeManual.class})
+public class IoTDBPipeLogicalBackupIT extends AbstractPipeDualTreeModelManualIT {
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Override
+ protected void setupConfig() {
+ super.setupConfig();
+ for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) {
+ env.getConfig().getConfigNodeJVMConfig().setInitHeapSize(128).setMaxHeapSize(256);
+ env.getConfig().getDataNodeJVMConfig().setInitHeapSize(256).setMaxHeapSize(768);
+ }
+ }
+
+ @Test
+ public void testBoundedContinuousTableCapture() throws Exception {
+ try (final Connection connection = senderEnv.getTableConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("create database backup_db");
+ statement.execute("use backup_db");
+ statement.execute("create table measurements(device STRING TAG, reading INT32 FIELD)");
+ statement.execute("create table other_table(device STRING TAG, reading INT32 FIELD)");
+ statement.execute("insert into measurements(time,device,reading) values(1,'d1',10)");
+ statement.execute("insert into other_table(time,device,reading) values(1,'d1',100)");
+ statement.execute("flush");
+ }
+ final String pipeName = "logical_backup_continuous_table";
+ final CompletableFuture writes =
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try (final Connection connection = senderEnv.getConnection();
+ final Statement statement = connection.createStatement();
+ final ResultSet result =
+ statement.executeQuery("show pipe " + pipeName)) {
+ return result.next();
+ }
+ });
+ try (final Connection connection = senderEnv.getTableConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("use backup_db");
+ statement.execute(
+ "insert into measurements(time,device,reading) values(2,'d1',20)");
+ statement.execute(
+ "insert into measurements(time,device,reading) values(4,'d1',40)");
+ statement.execute(
+ "insert into other_table(time,device,reading) values(2,'d1',200)");
+ statement.execute("delete from measurements where time <= 1");
+ }
+ } catch (final Exception e) {
+ throw new CompletionException(e);
+ }
+ });
+ final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath();
+ runConnected(
+ senderEnv,
+ "dump",
+ "--backup-dir",
+ parent.toString(),
+ "--backup-id",
+ "bounded",
+ "--pipe-name",
+ pipeName,
+ "--scope",
+ "schema,data",
+ "--model",
+ "table",
+ "--include",
+ "backup_db/measurements",
+ "--start-time",
+ "0",
+ "--end-time",
+ "3",
+ "--mode",
+ "continuous",
+ "--until",
+ Instant.now().plusSeconds(20).toString(),
+ "--timeout-seconds",
+ "180",
+ "--poll-interval-seconds",
+ "1");
+ writes.get(30, TimeUnit.SECONDS);
+ final Path backup = parent.resolve("bounded");
+ final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json"));
+ Assert.assertEquals("SEALED", catalog.status);
+ Assert.assertEquals("continuous", catalog.mode);
+ assertLogicalRequests(backup);
+ runConnected(receiverEnv, "restore", "--input", backup.toString());
+ try (final Connection connection = receiverEnv.getTableConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("use backup_db");
+ try (final ResultSet result = statement.executeQuery("select * from measurements")) {
+ Assert.assertTrue(result.next());
+ Assert.assertEquals(2L, result.getLong("time"));
+ Assert.assertEquals("d1", result.getString("device"));
+ Assert.assertEquals(20, result.getInt("reading"));
+ Assert.assertFalse(result.next());
+ }
+ try (final ResultSet result = statement.executeQuery("show tables")) {
+ while (result.next()) {
+ Assert.assertNotEquals("other_table", result.getString(1));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDumpPackPreflightRestoreWithScopeAndTimeFilters() throws Exception {
+ TestUtils.executeNonQueries(
+ senderEnv,
+ Arrays.asList(
+ "create database root.backup",
+ "create aligned timeseries root.backup.d(s1 INT32, s2 BOOLEAN)",
+ "insert into root.backup.d(time,s1,s2) aligned values(1,10,true)",
+ "insert into root.backup.d(time,s2) aligned values(2,true)",
+ "insert into root.backup.d(time,s1,s2) aligned values(3,30,false)",
+ "insert into root.backup.d(time,s1,s2) aligned values(4,40,true)",
+ "create database root.unselected",
+ "create timeseries root.unselected.d.s with datatype=INT32,encoding=PLAIN",
+ "insert into root.unselected.d(time,s) values(2,100)",
+ "create user logical_backup_excluded_user 'test_password'",
+ "flush"),
+ null);
+ final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath();
+ runConnected(
+ senderEnv,
+ "dump",
+ "--backup-dir",
+ parent.toString(),
+ "--backup-id",
+ "snapshot",
+ "--scope",
+ "config,schema,data",
+ "--model",
+ "tree",
+ "--include",
+ "root.backup.**",
+ "--start-time",
+ "2",
+ "--end-time",
+ "3",
+ "--timeout-seconds",
+ "180",
+ "--poll-interval-seconds",
+ "1");
+ final Path backup = parent.resolve("snapshot");
+ final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json"));
+ Assert.assertEquals("SEALED", catalog.status);
+ Assert.assertFalse(catalog.includeAuth);
+ Assert.assertFalse(catalog.streams.isEmpty());
+ assertLogicalRequests(backup);
+ final Path zip = parent.resolve("snapshot.zip");
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", backup.toString(), "--json"}));
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {"pack", "--input", backup.toString(), "--output", zip.toString()}));
+ Assert.assertEquals(
+ 0,
+ PipeLogicalBackupTool.run(
+ new String[] {"restore", "--input", zip.toString(), "--dry-run"}));
+ runConnected(receiverEnv, "restore", "--input", zip.toString(), "--dry-run");
+ try (final Connection connection = receiverEnv.getConnection();
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("show timeseries root.backup.**")) {
+ Assert.assertFalse(result.next());
+ }
+ final Path checkpoint = parent.resolve("restore-checkpoint.json");
+ runConnected(
+ receiverEnv, "restore", "--input", zip.toString(), "--checkpoint", checkpoint.toString());
+ runConnected(
+ receiverEnv,
+ "restore",
+ "--input",
+ zip.toString(),
+ "--checkpoint",
+ checkpoint.toString(),
+ "--resume");
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv,
+ "select * from root.backup.**",
+ "Time,root.backup.d.s1,root.backup.d.s2,",
+ new HashSet<>(Arrays.asList("2,null,true,", "3,30,false,")));
+ try (final Connection connection = receiverEnv.getConnection();
+ final Statement statement = connection.createStatement()) {
+ try (final ResultSet result = statement.executeQuery("show timeseries root.unselected.**")) {
+ Assert.assertFalse(result.next());
+ }
+ try (final ResultSet result = statement.executeQuery("list user")) {
+ while (result.next()) {
+ for (int column = 1; column <= result.getMetaData().getColumnCount(); column++) {
+ Assert.assertNotEquals("logical_backup_excluded_user", result.getString(column));
+ }
+ }
+ }
+ }
+ }
+
+ private static void assertLogicalRequests(final Path backup) throws Exception {
+ final Set logicalRequests =
+ Set.of(
+ PipeRequestType.TRANSFER_CONFIG_PLAN.getType(),
+ PipeRequestType.TRANSFER_PLAN_NODE.getType(),
+ PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(),
+ PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType());
+ new LogicalBackupArchiveReader()
+ .read(backup, false).stream()
+ .flatMap(stream -> stream.getEventGroups().stream())
+ .flatMap(group -> group.getRequests().stream())
+ .forEach(
+ record -> Assert.assertTrue(logicalRequests.contains(record.getRequestType())));
+ }
+
+ private static void runConnected(final BaseEnv env, final String command, final String... options)
+ throws Exception {
+ final List args =
+ new ArrayList<>(
+ Arrays.asList(
+ command,
+ "--host",
+ env.getIP(),
+ "--port",
+ env.getPort(),
+ "--user",
+ "root",
+ "--password-stdin"));
+ args.addAll(Arrays.asList(options));
+ final InputStream previous = System.in;
+ try {
+ System.setIn(new ByteArrayInputStream("root\n".getBytes(StandardCharsets.UTF_8)));
+ Assert.assertEquals(0, PipeLogicalBackupTool.run(args.toArray(new String[0])));
+ } finally {
+ System.setIn(previous);
+ }
+ }
+}
diff --git a/iotdb-client/cli/LOGICAL_BACKUP.md b/iotdb-client/cli/LOGICAL_BACKUP.md
new file mode 100644
index 000000000000..2fa605314e42
--- /dev/null
+++ b/iotdb-client/cli/LOGICAL_BACKUP.md
@@ -0,0 +1,138 @@
+
+
+# Logical backup
+
+Use `tools/logical-backup.sh` on Linux/macOS or
+`tools/windows/logical-backup.bat` on Windows. `--help` lists the supported options.
+Commands exit with 0 on success and 1 on failure. Errors go to stderr; `--json`
+provides a machine-readable result on stdout.
+
+## Capture, check, transport, restore
+
+```sh
+tools/logical-backup.sh dump --host source-host --port 6667 --user root \
+ --password-stdin --backup-dir /mnt/shared-backups --backup-id migration_001 \
+ --scope config,schema,data --model tree,table --mode snapshot
+
+tools/logical-backup.sh inspect --input /mnt/shared-backups/migration_001 --json
+tools/logical-backup.sh verify --input /mnt/shared-backups/migration_001 --json
+tools/logical-backup.sh pack --input /mnt/shared-backups/migration_001 \
+ --output /archives/migration_001.zip
+
+# Local verification; no target connection is opened.
+tools/logical-backup.sh restore --input /archives/migration_001.zip --dry-run
+
+# Target version, precision, authentication and protocol capability checks.
+tools/logical-backup.sh restore --input /archives/migration_001.zip \
+ --host target-host --port 6667 --user root --password-stdin --dry-run
+
+tools/logical-backup.sh restore --input /archives/migration_001.zip \
+ --host target-host --port 6667 --user root --password-stdin \
+ --checkpoint /archives/migration_001.restore.json
+
+# After an interrupted restore, retain all connection and input options and add --resume.
+tools/logical-backup.sh restore --input /archives/migration_001.zip \
+ --host target-host --port 6667 --user root --password-stdin \
+ --checkpoint /archives/migration_001.restore.json --resume
+```
+
+Passwords are read from stdin or `--password-env NAME`. Plaintext password
+arguments are rejected. Manifests and checkpoints contain no connection secrets.
+
+`dump` generates a directory, and `pack` only packages an existing directory.
+`pack` rejects connection options. Output and checkpoint files must be outside
+the input backup, and existing output archives are never overwritten.
+
+All source ConfigNodes, DataNodes, and the CLI must access the same backup
+directory at the specified path. This is a shared filesystem workflow, not a
+remote download protocol. Do not modify a directory while it is being checked,
+packed or restored.
+
+## Scope and boundaries
+
+`--scope` is required and selects `config`, `schema`, `data`, or a comma-separated
+combination. Schema capture includes its ConfigRegion definitions, such as
+databases and tables. `--model` defaults to `tree,table`.
+
+Authorization objects are excluded by default. Capturing them requires
+`--scope config` (possibly combined with other scopes) and `--include-auth true`.
+Restoring them also requires `--include-auth true`; inspect/dry-run reports this
+selection. Legacy config archives are conservatively treated as containing
+authorization objects.
+
+Object filters require a single model. Tree data accepts source path inclusion
+and exclusion lists; when capturing config/schema, the source supports a single
+prefix or full path without exclusions. Table filters use
+`database-regex/table-regex`; exclusions must match either whole databases or
+whole table-name patterns. Unsupported combinations fail before task creation.
+Data `--start-time` and `--end-time` are inclusive and use source timestamp
+precision. These filters do not remove the schema needed to describe selected
+objects.
+
+`snapshot` captures finite historical streams. `continuous` requires an ISO-8601
+`--until` boundary. Each source stops accepting realtime work at its capture
+boundary, drains accepted work, and acknowledges completion before task removal.
+The boundary uses source clocks; streams do not represent a global transaction.
+`--stop-sequence` is reserved and rejected because this server does not expose a
+stable global source sequence. `sourceHighWatermark` records each stream's durable
+archive sequence; it is not a cluster-wide source log position.
+
+The root `manifest.json` is the portable entry point. It declares scope,
+models, filters, capture settings, streams, relative file paths, full-file SHA-256
+digests, counts, status and capture/seal times. Stream manifests describe the
+versioned `.pwal` frames, CRCs, segment digests and committed event groups. Legacy
+stream-only directories remain readable; packing them adds a root manifest
+without changing the input.
+
+Config and Schema snapshots are decoded into logical plans before capture;
+data files are decoded into logical tablet requests. Each Region has its own
+ordered stream and writer. Pipe parameter validation creates no backup files.
+
+## Interrupted work and conflicts
+
+`dump --resume fail-if-exists` is the default. `new` chooses a new backup identity
+if the requested directory exists. `append` requires the same identity, scope
+and writing settings. A timed-out command reports the backup, directory and
+Pipe identity; query `SHOW PIPE` or rerun the same dump with `--resume append`.
+An already sealed matching backup is returned after verification.
+
+Writer sizes, fsync policy and batch/period combinations are validated before
+writing. Unknown events fail by default; `--on-unsupported skip` records a gap
+and seals the result as `INCOMPLETE`. Inspect, verify, pack and restore reject
+incomplete backups unless `--allow-incomplete` is explicit. That flag never
+bypasses checksum errors, truncated segments, missing files or invalid groups.
+
+Restore validates locally before connecting, then replays config, schema and
+data, sorting equal-type streams by stream ID. Dry-run sends no replay requests.
+Target preflight checks version, precision, authentication and protocol
+handshakes; object conflicts and object-level permissions are checked during
+replay. Only `--conflict fail` is supported. Target errors stop replay and retain
+the checkpoint; no schema is silently deleted, overwritten or skipped.
+
+Checkpoints bind the archive/content digest, backup format and identity, target
+host/port/user, completed groups and the next request within a group. A request
+that may have succeeded immediately before an interrupted checkpoint update can
+be retried; only that uncertain request accepts an idempotent-conflict response.
+There is no transaction across requests or streams. Completed checkpoints remain
+available for audit and repeat invocation.
+
+`export` and `import` are deprecated compatibility aliases. Offline `export` is
+`pack`; connected `export` performs `dump` then `pack`; `import` is `restore`.
+Both aliases print a migration notice on stderr.
diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
index 1316415682f5..9f49fc238e6e 100644
--- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -169,4 +169,44 @@ private CliMessages() {}
EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 =
"Logical backup option --%s has invalid identifier: %s";
+ public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D =
+ "export is deprecated; use dump to generate a backup and pack to create a ZIP.";
+ public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE =
+ "import is deprecated; use restore.";
+ public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE =
+ "Logical backup: %s";
+ public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 =
+ "Logical backup packed from %s to %s";
+public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E =
+ "--stop-sequence requires a stable source sequence capability; use --until with this server version.";
+ public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E =
+ "Object filters require --model tree or --model table; split mixed-model filtered dumps into separate backups.";
+ public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 =
+ "Table filters use database-regex/table-regex; exclusions must use .* for either database or table.";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 =
+ "Invalid logical backup dump option: --%s";
+ public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 =
+ "Invalid or conflicting logical backup arguments.";
+ public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E =
+ "Only --conflict fail is supported; restore never implicitly overwrites or deletes target objects.";
+ public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F =
+ "The existing backup task does not match this backup identity or capture options.";
+ public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 =
+ "Another dump process is managing this backup.";
+ public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 =
+ "Backup %s, task %s, directory %s: %s. The task is retained; query SHOW PIPE or repeat dump with the same options and --resume append.";
+ public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 =
+ "The backup task has not produced sealed streams yet; resume the same task to finish sealing.";
+ public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D =
+ "An existing checkpoint requires --resume; --resume requires an existing checkpoint.";
+ public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB =
+ "The target timestamp precision does not match the backup.";
+ public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C =
+ "The target version or data model is incompatible with the backup.";
+ public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 =
+ "The checkpoint must be outside the input backup.";
+ public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 =
+ "Restoring authorization objects requires --include-auth true; inspect the backup or use --dry-run first.";
+ public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 =
+ "logical-backup [options]\n dump Connect to a source and seal a backup directory; does not create a ZIP.\n inspect Show backup identity, scope, streams, segments, counts and status.\n verify Check structure, checksums and semantics without changing the input.\n pack Verify and package an existing directory as ZIP; accepts no connection options.\n restore Preflight and replay a directory, root manifest.json or ZIP.\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous requires --until; --stop-sequence is reserved and currently rejected.\n Object filters require one model: Tree uses path lists; Table uses database-regex/table-regex.\n Tree config/schema capture accepts a single prefix/full path without exclusions.\n Table exclusions must select whole databases or whole table-name patterns.\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n Restoring authorization objects requires --include-auth true (legacy config is conservative).\n A target connection is required for replay; local dry-run needs no connection.\n Target preflight checks version, precision, authentication and handshake support;\n object conflicts and per-object privileges are checked during replay.\n\nDeprecated aliases: export --input DIR --output ZIP = pack;\n export with source connection options = dump then pack; import = restore.\nExit code: 0 success, 1 failure. Errors and deprecation notices go to stderr.";
}
diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
index c26071d0bf5f..b8733d990479 100644
--- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
+++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java
@@ -165,4 +165,44 @@ private CliMessages() {}
public static final String
EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 =
"逻辑备份选项 --%s 的标识符无效：%s";
+ public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D =
+ "export 已弃用；请使用 dump 生成备份，使用 pack 创建 ZIP。";
+ public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE =
+ "import 已弃用；请使用 restore。";
+ public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE =
+ "逻辑备份：%s";
+ public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 =
+ "已将逻辑备份从 %s 打包至 %s";
+public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E =
+ "--stop-sequence 需要源端提供稳定序号能力；当前服务端版本请使用 --until。";
+ public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E =
+ "对象过滤需要 --model tree 或 --model table；请将混合模型过滤拆为独立备份。";
+ public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 =
+ "Table 过滤使用数据库正则/表名正则；排除过滤中数据库或表名必须有一项为 .*。";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 =
+ "逻辑备份 dump 选项无效：--%s";
+ public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 =
+ "逻辑备份参数无效或相互冲突。";
+ public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E =
+ "当前仅支持 --conflict fail；restore 不会隐式覆盖或删除目标对象。";
+ public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F =
+ "已有备份任务与本次备份身份或捕获选项不一致。";
+ public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 =
+ "其他 dump 进程正在管理此备份。";
+ public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 =
+ "备份 %s，任务 %s，目录 %s：%s。任务已保留；可使用 SHOW PIPE 查询，或使用相同选项和 --resume append 继续 dump。";
+ public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 =
+ "备份任务尚未产出封存的流；请继续原任务以完成封存。";
+ public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D =
+ "使用已有 checkpoint 必须指定 --resume；--resume 要求 checkpoint 已存在。";
+ public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB =
+ "目标时间精度与备份不一致。";
+ public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C =
+ "目标版本或数据模型与备份不兼容。";
+ public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 =
+ "checkpoint 必须位于输入备份之外。";
+ public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 =
+ "恢复权限对象必须显式指定 --include-auth true；请先 inspect 备份或执行 --dry-run。";
+ public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 =
+ "logical-backup [选项]\n dump 连接源端并封存备份目录，不生成 ZIP。\n inspect 查看备份身份、范围、流、segment、计数和状态。\n verify 检查结构、校验和与语义，不修改输入。\n pack 校验已有目录并打包为 ZIP，不接受数据库连接参数。\n restore 预检并回放备份目录、根 manifest.json 或 ZIP。\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous 必须提供 --until；--stop-sequence 是预留参数，当前会被拒绝。\n 对象过滤只允许选择一个模型：Tree 使用路径列表，Table 使用数据库正则/表名正则。\n Tree config/schema 捕获只接受单个前缀或完整路径，不支持排除过滤。\n Table 排除过滤必须覆盖整个数据库或整个表名模式。\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n 恢复权限对象必须指定 --include-auth true；旧 config 备份按包含权限对象处理。\n 实际恢复需要目标连接；本地 dry-run 无需连接。\n 目标预检检查版本、时间精度、认证和握手能力；\n 对象冲突及对象级权限在回放时检查。\n\n弃用别名：export --input DIR --output ZIP = pack；\n 带源端连接参数的 export = dump 后接 pack；import = restore。\n退出码：0 成功，1 失败。错误及弃用提示输出到标准错误。";
}
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java
new file mode 100644
index 000000000000..e28e288df0b4
--- /dev/null
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool.pipe;
+
+import org.apache.iotdb.cli.i18n.CliMessages;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import com.google.gson.Gson;
+import org.apache.commons.cli.CommandLine;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/** Converts the public dump contract to source and sink parameters before any remote mutation. */
+final class LogicalBackupDumpOptions {
+
+ final LogicalBackupCatalog catalog = new LogicalBackupCatalog();
+ final Map source = new LinkedHashMap<>();
+ final Map sink = new LinkedHashMap<>();
+ final String resume;
+ final long until;
+
+ LogicalBackupDumpOptions(final CommandLine line, final boolean legacy) throws IOException {
+ catalog.scope =
+ choices(
+ line.getOptionValue("scope", legacy ? "config,schema,data" : ""),
+ List.of("config", "schema", "data"),
+ "scope");
+ catalog.model =
+ choices(line.getOptionValue("model", "tree,table"), List.of("tree", "table"), "model");
+ catalog.mode = line.getOptionValue("mode", "snapshot");
+ if (!List.of("snapshot", "continuous").contains(catalog.mode)) {
+ throw invalid("mode");
+ }
+ final String includeAuth = line.getOptionValue("include-auth", "false");
+ if (!List.of("true", "false").contains(includeAuth)) {
+ throw invalid("include-auth");
+ }
+ catalog.includeAuth = Boolean.parseBoolean(includeAuth);
+ if (catalog.includeAuth && !catalog.scope.contains("config")) {
+ throw invalid("include-auth");
+ }
+ if (line.hasOption("stop-sequence")) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E);
+ }
+ if ("continuous".equals(catalog.mode)) {
+ if (!line.hasOption("until")) {
+ throw invalid("until");
+ }
+ try {
+ until = Instant.parse(line.getOptionValue("until")).toEpochMilli();
+ } catch (final java.time.DateTimeException e) {
+ throw invalid("until");
+ }
+ if (until <= System.currentTimeMillis() && !"append".equals(line.getOptionValue("resume"))) {
+ throw invalid("until");
+ }
+ } else {
+ if (line.hasOption("until")) {
+ throw invalid("until");
+ }
+ until = -1;
+ }
+ resume = line.getOptionValue("resume", "fail-if-exists");
+ if (!List.of("fail-if-exists", "append", "new").contains(resume)) {
+ throw invalid("resume");
+ }
+ catalog.captureOptions.put("scope", String.join(",", catalog.scope));
+ catalog.captureOptions.put("model", String.join(",", catalog.model));
+ catalog.captureOptions.put("mode", catalog.mode);
+ catalog.captureOptions.put("include-auth", includeAuth);
+ for (final String key : List.of("include", "exclude", "start-time", "end-time", "until")) {
+ if (line.hasOption(key)) {
+ catalog.filters.put(key, line.getOptionValue(key));
+ catalog.captureOptions.put(key, line.getOptionValue(key));
+ }
+ }
+ catalog.filters.put("time-boundary", "closed-closed");
+ final List inclusion = new ArrayList<>();
+ if (catalog.scope.contains("config")) {
+ inclusion.add("schema.database");
+ inclusion.add("schema.ttl");
+ inclusion.add("schema.timeseries.template.create");
+ inclusion.add("schema.timeseries.template.set");
+ inclusion.add("schema.timeseries.template.unset");
+ inclusion.add("schema.timeseries.template.alter");
+ inclusion.add("schema.timeseries.template.drop");
+ }
+ if (catalog.scope.contains("schema")) {
+ inclusion.add("schema");
+ }
+ if (catalog.scope.contains("data")) {
+ inclusion.add("data");
+ }
+ if (catalog.includeAuth) {
+ inclusion.add("auth");
+ }
+ source.put("source", "iotdb-source");
+ source.put("source.inclusion", String.join(",", inclusion));
+ source.put("source.inclusion.exclusion", catalog.includeAuth ? "" : "auth");
+ source.put("source.skipif", "");
+ source.put("source.capture.tree", Boolean.toString(catalog.model.contains("tree")));
+ source.put("source.capture.table", Boolean.toString(catalog.model.contains("table")));
+ source.put("source.mode", "snapshot".equals(catalog.mode) ? "snapshot" : "live");
+ source.put("source.realtime.enable", Boolean.toString("continuous".equals(catalog.mode)));
+ source.put("source.logical-backup.snapshot", Boolean.toString("snapshot".equals(catalog.mode)));
+ if (until > 0) {
+ source.put("source.logical-backup.until", Long.toString(until));
+ source.put("source.realtime.mode", "forced-log");
+ }
+ // Decode TsFiles so the source applies exact path/time filters to logical rows.
+ source.put("source.mods.enable", "true");
+ for (final String key : List.of("start-time", "end-time")) {
+ if (line.hasOption(key)) {
+ source.put("source." + key, line.getOptionValue(key));
+ }
+ }
+ applyObjectFilters(line);
+ sink.put("sink", "logical-backup-sink");
+ sink.put("sink.resume", "append");
+ final Map defaults = new LinkedHashMap<>();
+ defaults.put("segment-size-bytes", "268435456");
+ defaults.put("max-record-bytes", "67108864");
+ defaults.put("fsync-policy", "always");
+ defaults.put("on-unsupported", "fail");
+ defaults.put("include-heartbeat", "false");
+ for (final Map.Entry entry : defaults.entrySet()) {
+ catalog.captureOptions.put(
+ entry.getKey(), line.getOptionValue(entry.getKey(), entry.getValue()));
+ sink.put(
+ "sink."
+ + ("on-unsupported".equals(entry.getKey()) ? "on-unsupported-event" : entry.getKey()),
+ catalog.captureOptions.get(entry.getKey()));
+ }
+ for (final String key : List.of("fsync-batch-operations", "fsync-period-ms")) {
+ if (line.hasOption(key)) {
+ sink.put("sink." + key, line.getOptionValue(key));
+ catalog.captureOptions.put(key, line.getOptionValue(key));
+ }
+ }
+ sink.put("sink.capture-options", new Gson().toJson(catalog.captureOptions));
+ // Reuse the same validation as all three server-side writers.
+ final Map validation = new LinkedHashMap<>(sink);
+ validation.put("sink.dir", line.getOptionValue("backup-dir", "."));
+ new LogicalBackupSettings(new PipeParameters(validation), "backup");
+ }
+
+ private void applyObjectFilters(final CommandLine line) throws IOException {
+ if (!line.hasOption("include") && !line.hasOption("exclude")) {
+ return;
+ }
+ if (catalog.model.size() != 1) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E);
+ }
+ if (catalog.model.contains("tree")) {
+ if (line.hasOption("include")) {
+ source.put("source.path.inclusion", line.getOptionValue("include"));
+ }
+ if (line.hasOption("exclude")) {
+ source.put("source.path.exclusion", line.getOptionValue("exclude"));
+ }
+ final TreePattern pattern =
+ TreePattern.parsePipePatternFromSourceParameters(new PipeParameters(source));
+ if (!pattern.isLegal()
+ || ((catalog.scope.contains("config") || catalog.scope.contains("schema"))
+ && !(pattern instanceof IoTDBTreePatternOperations
+ && ((IoTDBTreePatternOperations) pattern).isPrefixOrFullPath()))) {
+ throw invalid("include/exclude");
+ }
+ return;
+ }
+ // TablePattern accepts independent database/table regular expressions. Reject correlated
+ // exclusions rather than silently backing up their Cartesian product.
+ final String[] include = tableFilter(line.getOptionValue("include", ".*/.*"));
+ final String[] exclude =
+ line.hasOption("exclude") ? tableFilter(line.getOptionValue("exclude")) : null;
+ if (exclude != null && !".*".equals(exclude[0]) && !".*".equals(exclude[1])) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23);
+ }
+ for (int index = 0; index < 2; index++) {
+ String regex = include[index];
+ if (exclude != null && !".*".equals(exclude[index])) {
+ regex = "(?!(?:" + exclude[index] + ")$)(?:" + regex + ")";
+ }
+ source.put(index == 0 ? "source.database-name" : "source.table-name", regex);
+ }
+ }
+
+ private static String[] tableFilter(final String filter) throws IOException {
+ final String[] parts = filter.split("/", -1);
+ if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23);
+ }
+ try {
+ Pattern.compile(parts[0]);
+ Pattern.compile(parts[1]);
+ } catch (final java.util.regex.PatternSyntaxException e) {
+ throw invalid("include/exclude");
+ }
+ return parts;
+ }
+
+ private static List choices(
+ final String value, final List allowed, final String key) throws IOException {
+ final List values = Arrays.asList(value.split(",", -1));
+ if (values.isEmpty()
+ || !allowed.containsAll(values)
+ || values.stream().distinct().count() != values.size()) {
+ throw invalid(key);
+ }
+ // Canonical ordering makes equivalent requests compare equal when resuming.
+ final List canonical = new ArrayList<>(allowed);
+ canonical.retainAll(values);
+ return canonical;
+ }
+
+ String createSql(final String pipeName, final String root, final String backupId) {
+ final Map sinkOptions = new LinkedHashMap<>(sink);
+ sinkOptions.put("sink.dir", root);
+ sinkOptions.put("sink.backup-id", backupId);
+ return "CREATE PIPE "
+ + pipeName
+ + " WITH SOURCE ("
+ + sqlOptions(source)
+ + ") WITH SINK ("
+ + sqlOptions(sinkOptions)
+ + ")";
+ }
+
+ private static String sqlOptions(final Map options) {
+ final List values = new ArrayList<>();
+ options.forEach((key, value) -> values.add("'" + key + "'='" + value.replace("'", "''") + "'"));
+ return String.join(",", values);
+ }
+
+ private static IOException invalid(final String key) {
+ return new IOException(
+ String.format(CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0, key));
+ }
+}
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
index 631353027a61..e73067ed69ea 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java
@@ -25,9 +25,11 @@
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req;
@@ -43,7 +45,6 @@
import com.google.gson.JsonParseException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
@@ -53,7 +54,6 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
-import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -73,13 +73,16 @@
import java.util.Set;
import java.util.UUID;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */
public final class PipeLogicalBackupTool {
private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+ private static final String COMMAND_DUMP = "dump";
+ private static final String COMMAND_PACK = "pack";
+ private static final String COMMAND_RESTORE = "restore";
private static final String COMMAND_INSPECT = "inspect";
private static final String COMMAND_VERIFY = "verify";
private static final String COMMAND_EXPORT = "export";
@@ -129,71 +132,199 @@ public static void main(final String[] args) {
}
}
- static int run(final String[] args) throws Exception {
+ public static int run(final String[] args) throws Exception {
if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) {
printUsage();
return 0;
}
final String command = args[0].toLowerCase(java.util.Locale.ROOT);
+ if (!List.of(
+ COMMAND_DUMP,
+ COMMAND_PACK,
+ COMMAND_RESTORE,
+ COMMAND_INSPECT,
+ COMMAND_VERIFY,
+ COMMAND_EXPORT,
+ COMMAND_IMPORT)
+ .contains(command)) {
+ throw new ParseException(
+ String.format(
+ CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command));
+ }
+ if (java.util.Arrays.asList(args).contains("--help")
+ || java.util.Arrays.asList(args).contains("-h")) {
+ printUsage();
+ return 0;
+ }
final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length));
switch (command) {
case COMMAND_INSPECT:
return inspect(line);
case COMMAND_VERIFY:
return verify(line);
+ case COMMAND_DUMP:
+ final Path dumped = dump(line, false);
+ printDumpResult(line, dumped);
+ return 0;
+ case COMMAND_PACK:
+ return pack(line);
+ case COMMAND_RESTORE:
+ return importBackup(line);
case COMMAND_EXPORT:
- return export(line);
+ System.err.println(
+ CliMessages
+ .LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D);
+ if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) {
+ final Path source = dump(line, true);
+ final List streams =
+ new LogicalBackupArchiveReader()
+ .read(source, line.hasOption(OPTION_ALLOW_INCOMPLETE));
+ return exportArchive(
+ source,
+ Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)),
+ streams,
+ line.hasOption(OPTION_JSON));
+ }
+ return pack(line);
case COMMAND_IMPORT:
+ System.err.println(CliMessages.LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE);
return importBackup(line);
default:
- throw new ParseException(
- String.format(
- CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command));
+ throw new AssertionError(command);
}
}
- private static CommandLine parse(final String command, final String[] args)
- throws ParseException {
+ static CommandLine parse(final String command, final String[] args) throws ParseException {
final Options options = new Options();
- addOption(options, OPTION_INPUT, true, false);
- addOption(options, OPTION_OUTPUT, true, false);
- addOption(options, OPTION_RESUME, true, false);
- addOption(options, OPTION_FORMAT, true, false);
- addOption(options, OPTION_DEEP, false, false);
- addOption(options, OPTION_SOURCE, true, false);
- addOption(options, OPTION_TARGET, true, false);
- addOption(options, OPTION_HOST, true, false);
- addOption(options, OPTION_PORT, true, false);
- addOption(options, OPTION_USER, true, false);
- addOption(options, OPTION_PASSWORD_STDIN, false, false);
- addOption(options, OPTION_PASSWORD_ENV, true, false);
- addOption(options, OPTION_CHECKPOINT, true, false);
- addOption(options, OPTION_DRY_RUN, false, false);
- addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
- addOption(options, OPTION_JSON, false, false);
- addOption(options, OPTION_BACKUP_DIR, true, false);
- addOption(options, OPTION_PIPE_NAME, true, false);
- addOption(options, OPTION_BACKUP_ID, true, false);
- addOption(options, OPTION_TIMEOUT_SECONDS, true, false);
- addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false);
+ final Set names = new LinkedHashSet<>(List.of(OPTION_JSON, OPTION_ALLOW_INCOMPLETE));
+ final boolean legacyExport = COMMAND_EXPORT.equals(command);
+ final boolean legacyImport = COMMAND_IMPORT.equals(command);
+ final boolean dump = COMMAND_DUMP.equals(command) || legacyExport;
+ final boolean restore = COMMAND_RESTORE.equals(command) || legacyImport;
+ if (!COMMAND_DUMP.equals(command)) {
+ names.add(OPTION_INPUT);
+ }
+ if (COMMAND_PACK.equals(command) || legacyExport) {
+ names.add(OPTION_OUTPUT);
+ }
+ if (dump || restore) {
+ names.addAll(
+ List.of(
+ OPTION_HOST,
+ OPTION_PORT,
+ OPTION_USER,
+ OPTION_PASSWORD_STDIN,
+ OPTION_PASSWORD_ENV,
+ OPTION_RESUME));
+ }
+ if (restore) {
+ names.addAll(List.of(OPTION_CHECKPOINT, OPTION_DRY_RUN, "conflict", "include-auth"));
+ }
+ if (dump) {
+ names.addAll(
+ List.of(
+ OPTION_BACKUP_DIR,
+ OPTION_BACKUP_ID,
+ OPTION_PIPE_NAME,
+ OPTION_TIMEOUT_SECONDS,
+ OPTION_POLL_INTERVAL_SECONDS,
+ "scope",
+ "model",
+ "include",
+ "exclude",
+ "start-time",
+ "end-time",
+ "include-auth",
+ "mode",
+ "until",
+ "stop-sequence",
+ "segment-size-bytes",
+ "max-record-bytes",
+ "fsync-policy",
+ "fsync-batch-operations",
+ "fsync-period-ms",
+ "on-unsupported",
+ "include-heartbeat"));
+ }
+ if (legacyExport || legacyImport) {
+ names.add(OPTION_SOURCE);
+ if (legacyExport) {
+ names.addAll(List.of(OPTION_TARGET, OPTION_FORMAT));
+ }
+ }
+ if (COMMAND_VERIFY.equals(command)) {
+ names.add(OPTION_DEEP); // Accepted for compatibility; verification is always deep.
+ }
+ final Set flags =
+ Set.of(
+ OPTION_JSON,
+ OPTION_ALLOW_INCOMPLETE,
+ OPTION_DRY_RUN,
+ OPTION_PASSWORD_STDIN,
+ OPTION_DEEP);
+ for (final String name : names) {
+ if (OPTION_RESUME.equals(name) && restore) {
+ options.addOption(
+ Option.builder().longOpt(name).hasArg(legacyImport).optionalArg(legacyImport).build());
+ } else {
+ addOption(options, name, !flags.contains(name), false);
+ }
+ }
final CommandLine line = new DefaultParser().parse(options, args);
- final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE);
- if ((input == null || input.isBlank())
- && (!COMMAND_EXPORT.equals(command) || !line.hasOption(OPTION_BACKUP_DIR))) {
+ if (!line.getArgList().isEmpty()
+ || (line.hasOption(OPTION_INPUT) && line.hasOption(OPTION_SOURCE))
+ || (line.hasOption(OPTION_OUTPUT) && line.hasOption(OPTION_TARGET))) {
+ throw new ParseException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ if (!COMMAND_DUMP.equals(command)
+ && !(legacyExport && line.hasOption(OPTION_BACKUP_DIR))
+ && optionValue(line, OPTION_INPUT, OPTION_SOURCE) == null) {
throw new ParseException(
String.format(
CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT));
}
- final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET);
- if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) {
+ if ((COMMAND_PACK.equals(command) || legacyExport)
+ && optionValue(line, OPTION_OUTPUT, OPTION_TARGET) == null) {
throw new ParseException(
- CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1);
+ String.format(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_OUTPUT));
}
- final String format = line.getOptionValue(OPTION_FORMAT, "binary");
- if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) {
+ if (legacyExport && !"binary".equals(line.getOptionValue(OPTION_FORMAT, "binary"))) {
throw new ParseException(
String.format(
- CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format));
+ CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1,
+ line.getOptionValue(OPTION_FORMAT)));
+ }
+ if (line.hasOption(OPTION_PORT)) {
+ final long port = longOption(line, OPTION_PORT, 6667);
+ if (port <= 0 || port > 65535) {
+ throw new ParseException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ }
+ if (restore && !"fail".equals(line.getOptionValue("conflict", "fail"))) {
+ throw new ParseException(
+ CliMessages
+ .EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E);
+ }
+ // Offline export is exactly pack, including its refusal of source connection options.
+ if (legacyExport && optionValue(line, OPTION_INPUT, OPTION_SOURCE) != null) {
+ final Set packOptions =
+ Set.of(
+ OPTION_INPUT,
+ OPTION_SOURCE,
+ OPTION_OUTPUT,
+ OPTION_TARGET,
+ OPTION_FORMAT,
+ OPTION_JSON,
+ OPTION_ALLOW_INCOMPLETE);
+ for (final Option option : line.getOptions()) {
+ if (!packOptions.contains(option.getLongOpt())) {
+ throw new ParseException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ }
}
return line;
}
@@ -203,128 +334,228 @@ private static void addOption(
options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build());
}
- private static int inspect(final CommandLine line) throws IOException {
- final List streams = read(line);
- if (line.hasOption(OPTION_JSON)) {
- final List> output = new ArrayList<>();
- for (final BackupStream stream : streams) {
- final Map item = new HashMap<>();
- item.put("manifest", stream.getManifest());
- item.put("recordCount", stream.getRecords().size());
- item.put("eventGroupCount", stream.getEventGroups().size());
- output.add(item);
- }
- System.out.println(GSON.toJson(output));
- return 0;
+ private static Map summary(final List streams) {
+ final Map result = new LinkedHashMap<>();
+ if (!streams.isEmpty() && streams.get(0).getCatalog() != null) {
+ result.put("manifest", streams.get(0).getCatalog());
}
+ final List> details = new ArrayList<>();
for (final BackupStream stream : streams) {
- System.out.println(
- String.format(
- CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
- stream.getManifest().streamId,
- stream.getRecords().size(),
- stream.getEventGroups().size()));
+ final Map item = new LinkedHashMap<>();
+ item.put("manifest", stream.getManifest());
+ item.put("recordCount", stream.getRecords().size());
+ item.put("eventGroupCount", stream.getEventGroups().size());
+ details.add(item);
}
+ result.put("streams", details);
+ return result;
+ }
+
+ private static int inspect(final CommandLine line) throws IOException {
+ final List streams = read(line);
+ printSummary(line, summary(streams));
return 0;
}
private static int verify(final CommandLine line) throws IOException {
final List streams = read(line);
- long recordCount = 0;
- long eventCount = 0;
- for (final BackupStream stream : streams) {
- recordCount += stream.getRecords().size();
- eventCount += stream.getEventGroups().size();
- }
- System.out.println(
- String.format(
- CliMessages
- .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B,
- streams.size(),
- recordCount,
- eventCount));
+ final Map result = summary(streams);
+ result.put("structural", "passed");
+ result.put("checksum", "passed");
+ result.put("semantic", "passed");
+ printSummary(line, result);
return 0;
}
- private static int export(final CommandLine line) throws Exception {
- if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) {
- return managedExport(line);
+ private static void printSummary(final CommandLine line, final Map result) {
+ if (line.hasOption(OPTION_JSON)) {
+ System.out.println(GSON.toJson(result));
+ } else {
+ System.out.println(
+ String.format(CliMessages.LOG_LOGICAL_BACKUP_ARG_1E5C23CE, GSON.toJson(result)));
}
+ }
+
+ private static int pack(final CommandLine line) throws IOException {
final Path source = Path.of(input(line)).toAbsolutePath().normalize();
final Path target =
Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
final List streams = read(line);
- return exportArchive(source, target, streams);
+ return exportArchive(source, target, streams, line.hasOption(OPTION_JSON));
}
- private static int managedExport(final CommandLine line) throws Exception {
+ private static Path dump(final CommandLine line, final boolean legacy) throws Exception {
+ final LogicalBackupDumpOptions options = new LogicalBackupDumpOptions(line, legacy);
final String host = required(line, OPTION_HOST);
final int port = Integer.parseInt(required(line, OPTION_PORT));
final String user = line.getOptionValue(OPTION_USER, "root");
- final String password = password(line);
final Path backupRoot = Path.of(required(line, OPTION_BACKUP_DIR)).toAbsolutePath().normalize();
- final Path target =
- Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize();
- final String pipeName =
- line.getOptionValue(
- OPTION_PIPE_NAME,
- "logical_backup_export_" + UUID.randomUUID().toString().replace('-', '_'));
- validateIdentifier(pipeName, OPTION_PIPE_NAME);
- final String backupId = line.getOptionValue(OPTION_BACKUP_ID, pipeName);
- validateIdentifier(backupId, OPTION_BACKUP_ID);
+ String backupId =
+ legacy
+ ? line.getOptionValue(
+ OPTION_BACKUP_ID, "backup_" + UUID.randomUUID().toString().replace('-', '_'))
+ : required(line, OPTION_BACKUP_ID);
+ LogicalBackupSettings.validateBackupId(backupId);
+ if ("new".equals(options.resume)
+ && Files.exists(backupRoot.resolve(backupId), LinkOption.NOFOLLOW_LINKS)) {
+ backupId += "-new-" + UUID.randomUUID().toString().substring(0, 8);
+ LogicalBackupSettings.validateBackupId(backupId);
+ }
+ final Path directory = backupRoot.resolve(backupId);
+ LogicalBackupCatalog.rejectSymbolicLinks(directory);
+ final boolean append = "append".equals(options.resume);
+ final boolean existed = Files.exists(directory, LinkOption.NOFOLLOW_LINKS);
+ if (existed && !append) {
+ throw new java.nio.file.FileAlreadyExistsException(directory.toString());
+ }
+ final String password = password(line);
final long timeoutSeconds = longOption(line, OPTION_TIMEOUT_SECONDS, 3600L);
final long pollIntervalSeconds = longOption(line, OPTION_POLL_INTERVAL_SECONDS, 2L);
- if (timeoutSeconds <= 0 || pollIntervalSeconds <= 0) {
+ if (timeoutSeconds <= 0
+ || timeoutSeconds > 31_536_000L
+ || pollIntervalSeconds <= 0
+ || pollIntervalSeconds > 3600L) {
throw new IOException(
CliMessages
.EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769);
}
-
- final String createSql =
- "CREATE PIPE "
- + pipeName
- + " WITH SOURCE ('source'='iotdb-source','source.mode'='snapshot','source.inclusion'='all',"
- + "'source.capture.tree'='true','source.capture.table'='true',"
- + "'source.realtime.enable'='false') WITH SINK ('sink'='logical-backup-sink','sink.dir'='"
- + sqlString(backupRoot.toString())
- + "','sink.backup-id'='"
- + sqlString(backupId)
- + "','sink.resume'='append')";
- boolean created = false;
- try (final Session session =
- new Session.Builder().host(host).port(port).username(user).password(password).build()) {
- session.open();
- session.executeNonQueryStatement(createSql);
- created = true;
- session.executeNonQueryStatement("START PIPE " + pipeName);
- waitForSnapshotPipe(session, pipeName, timeoutSeconds, pollIntervalSeconds);
- final Path source = backupRoot.resolve(backupId).normalize();
- final List streams = new LogicalBackupArchiveReader().read(source, false);
- return exportArchive(source, target, streams);
- } finally {
- if (created) {
- try (final Session cleanup =
- new Session.Builder().host(host).port(port).username(user).password(password).build()) {
- cleanup.open();
- try {
- cleanup.executeNonQueryStatement("STOP PIPE " + pipeName);
- } catch (final Exception ignored) {
- // A completed snapshot pipe may already have been auto-dropped.
- }
- try {
- cleanup.executeNonQueryStatement("DROP PIPE " + pipeName);
- } catch (final Exception ignored) {
- // Keep the original export failure when cleanup races with auto-drop.
- }
- } catch (final Exception ignored) {
- // The archive is complete once the generated pipe disappears; cleanup is best effort.
+ LogicalBackupCatalog catalog = options.catalog;
+ catalog.backupId = backupId;
+ catalog.pipeName =
+ line.getOptionValue(
+ OPTION_PIPE_NAME, "logical_backup_" + UUID.randomUUID().toString().replace('-', '_'));
+ validateIdentifier(catalog.pipeName, OPTION_PIPE_NAME);
+ if (existed) {
+ final LogicalBackupCatalog previous =
+ LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ if (previous == null
+ || !catalog.backupId.equals(previous.backupId)
+ || !catalog.captureOptions.equals(previous.captureOptions)
+ || (line.hasOption(OPTION_PIPE_NAME) && !catalog.pipeName.equals(previous.pipeName))) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F);
+ }
+ catalog = previous;
+ if ("SEALED".equals(catalog.status) || "INCOMPLETE".equals(catalog.status)) {
+ new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE));
+ return directory;
+ }
+ } else {
+ Files.createDirectories(backupRoot);
+ Files.createDirectory(directory);
+ }
+ // Keep the lock inode: deleting it after unlock can let two later callers lock different files.
+ final Path lock = directory.resolve(".backup.lock");
+ try (final java.nio.channels.FileChannel channel =
+ java.nio.channels.FileChannel.open(
+ lock, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ final java.nio.channels.FileLock ignored = channel.tryLock()) {
+ if (ignored == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828);
+ }
+ catalog.write(directory);
+ final String pipeName = catalog.pipeName;
+ try (final Session session =
+ new Session.Builder().host(host).port(port).username(user).password(password).build()) {
+ session.open();
+ final PipeStatus existing = readPipeStatus(session, pipeName);
+ if (existing != null && !append) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F);
}
+ if (existing == null && (!existed || streamManifests(directory).isEmpty())) {
+ session.executeNonQueryStatement(
+ options.createSql(pipeName, backupRoot.toString(), backupId));
+ session.executeNonQueryStatement("START PIPE " + pipeName);
+ }
+ waitForPipe(
+ session,
+ pipeName,
+ timeoutSeconds,
+ pollIntervalSeconds,
+ options.until,
+ line.hasOption(OPTION_JSON));
+ final List streams =
+ awaitSealedStreams(directory, Math.min(timeoutSeconds, 120L), pollIntervalSeconds);
+ catalog.seal(directory, streams);
+ } catch (final Exception e) {
+ throw new IOException(
+ String.format(
+ CliMessages
+ .EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87,
+ backupId,
+ pipeName,
+ directory,
+ e.getMessage()),
+ e);
}
}
+ new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE));
+ return directory;
+ }
+
+ private static List streamManifests(final Path root) throws IOException {
+ try (final java.util.stream.Stream paths = Files.walk(root)) {
+ return paths
+ .filter(path -> !path.equals(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)))
+ .filter(
+ path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString()))
+ .sorted()
+ .toList();
+ }
+ }
+
+ private static List readGeneratedStreams(final Path root) throws Exception {
+ final List streams = new ArrayList<>();
+ for (final Path manifest : streamManifests(root)) {
+ streams.addAll(new LogicalBackupArchiveReader().read(manifest, true));
+ }
+ if (streams.isEmpty()
+ || streams.stream()
+ .anyMatch(
+ stream ->
+ stream.getManifest().segments.stream()
+ .anyMatch(segment -> !"SEALED".equals(segment.status)))) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599);
+ }
+ return streams;
+ }
+
+ private static List awaitSealedStreams(
+ final Path root, final long timeoutSeconds, final long pollSeconds) throws Exception {
+ final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L;
+ Exception lastFailure;
+ do {
+ try {
+ return readGeneratedStreams(root);
+ } catch (final IOException e) {
+ lastFailure = e;
+ }
+ Thread.sleep(Math.min(pollSeconds * 1000L, 1000L));
+ } while (System.nanoTime() < deadline);
+ throw lastFailure;
+ }
+
+ private static void printDumpResult(final CommandLine line, final Path directory)
+ throws IOException {
+ final LogicalBackupCatalog catalog =
+ LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ final Map result = new LinkedHashMap<>();
+ result.put("backupId", catalog.backupId);
+ result.put("manifest", directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME).toString());
+ result.put("status", catalog.status);
+ result.put("sourceHighWatermark", catalog.sourceHighWatermark);
+ printSummary(line, result);
}
private static int exportArchive(
- final Path source, final Path target, final List streams) throws IOException {
+ final Path source, final Path target, final List streams, final boolean json)
+ throws IOException {
final Path normalizedSource = source.toAbsolutePath().normalize();
final Path normalizedTarget = target.toAbsolutePath().normalize();
final Path root;
@@ -340,7 +571,9 @@ private static int exportArchive(
.EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0,
normalizedSource));
}
- if (normalizedTarget.startsWith(root)) {
+ LogicalBackupCatalog.rejectSymbolicLinks(root);
+ LogicalBackupCatalog.rejectSymbolicLinks(normalizedTarget);
+ if (normalizedTarget.startsWith(root.toRealPath())) {
throw new IOException(
CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994);
}
@@ -352,16 +585,58 @@ private static int exportArchive(
Files.createDirectories(parent);
}
final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp");
+ final boolean incomplete =
+ streams.stream()
+ .anyMatch(
+ stream ->
+ stream.getManifest().skippedEventCount > 0
+ || "INCOMPLETE".equals(stream.getManifest().status)
+ || (stream.getCatalog() != null
+ && "INCOMPLETE".equals(stream.getCatalog().status)));
boolean published = false;
try {
+ final boolean legacy =
+ LogicalBackupCatalog.load(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)) == null;
+ final boolean relocateLegacyStream =
+ legacy
+ && streams.stream()
+ .anyMatch(stream -> stream.getManifestPath().getParent().equals(root));
try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) {
+ if (legacy) {
+ final LogicalBackupManifest first = streams.get(0).getManifest();
+ final LogicalBackupCatalog catalog = new LogicalBackupCatalog();
+ catalog.backupId = first.backupId;
+ catalog.pipeName = first.pipeName;
+ catalog.scope =
+ streams.stream().map(stream -> stream.getManifest().streamType).distinct().toList();
+ catalog.model = List.of("tree", "table");
+ catalog.includeAuth = containsConfigStream(streams);
+ catalog.captureOptions.putAll(first.captureOptions);
+ catalog.collect(root, streams);
+ if (relocateLegacyStream) {
+ for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) {
+ entry.manifest.file = "legacy-stream/" + entry.manifest.file;
+ for (final LogicalBackupCatalog.FileEntry segment : entry.segments) {
+ segment.file = "legacy-stream/" + segment.file;
+ }
+ }
+ }
+ zip.putNextEntry(new ZipEntry(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ zip.write(GSON.toJson(catalog).getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
for (final Path file : exportFiles(root, streams)) {
- final String entryName = root.relativize(file).toString().replace('\\', '/');
+ final String entryName =
+ (relocateLegacyStream ? "legacy-stream/" : "")
+ + root.relativize(file).toString().replace('\\', '/');
zip.putNextEntry(new ZipEntry(entryName));
Files.copy(file, zip);
zip.closeEntry();
}
}
+ try (final BackupInput packed = BackupInput.open(temporary)) {
+ new LogicalBackupArchiveReader().read(packed.getPath(), incomplete);
+ }
try (final java.nio.channels.FileChannel archiveChannel =
java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) {
archiveChannel.force(true);
@@ -377,19 +652,33 @@ private static int exportArchive(
Files.deleteIfExists(temporary);
}
}
- System.out.println(
- String.format(
- CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280,
- normalizedSource,
- normalizedTarget));
+ if (json) {
+ System.out.println(
+ GSON.toJson(
+ Map.of(
+ "input",
+ normalizedSource.toString(),
+ "output",
+ normalizedTarget.toString(),
+ "status",
+ incomplete ? "INCOMPLETE" : "SEALED")));
+ } else {
+ System.out.println(
+ String.format(
+ CliMessages.LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304,
+ normalizedSource,
+ normalizedTarget));
+ }
return 0;
}
- private static void waitForSnapshotPipe(
+ private static void waitForPipe(
final Session session,
final String pipeName,
final long timeoutSeconds,
- final long pollSeconds)
+ final long pollSeconds,
+ final long until,
+ final boolean json)
throws Exception {
final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L;
while (System.nanoTime() < deadline) {
@@ -397,14 +686,15 @@ private static void waitForSnapshotPipe(
if (status == null) {
return;
} else {
- System.out.println(
- String.format(
- CliMessages
- .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B,
- pipeName,
- status.state,
- status.remainingEvents,
- status.estimatedSeconds));
+ (json ? System.err : System.out)
+ .println(
+ String.format(
+ CliMessages
+ .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B,
+ pipeName,
+ status.state,
+ status.remainingEvents,
+ status.estimatedSeconds));
if (status.state != null
&& (status.state.equalsIgnoreCase("ERROR")
|| status.state.equalsIgnoreCase("STOPPED"))) {
@@ -488,6 +778,10 @@ private static final class PipeStatus {
private static List exportFiles(final Path root, final List streams)
throws IOException {
final Set files = new LinkedHashSet<>();
+ final Path rootManifest = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ if (LogicalBackupCatalog.load(rootManifest) != null) {
+ files.add(rootManifest);
+ }
for (final BackupStream stream : streams) {
final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize();
if (!manifestPath.startsWith(root)) {
@@ -510,25 +804,102 @@ private static List exportFiles(final Path root, final List
private static int importBackup(final CommandLine line) throws Exception {
final List streams = read(line);
- final boolean dryRun = line.hasOption(OPTION_DRY_RUN);
- if (dryRun) {
- System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B);
- for (final BackupStream stream : streams) {
- System.out.println(
- String.format(
- CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0,
- stream.getManifest().streamId,
- stream.getRecords().size(),
- stream.getEventGroups().size()));
- }
+ final Map report = summary(streams);
+ final boolean authorizationObjects =
+ streams.get(0).getCatalog() == null
+ ? containsConfigStream(streams)
+ : streams.get(0).getCatalog().includeAuth;
+ final String includeAuth = line.getOptionValue("include-auth", "false");
+ if (!List.of("true", "false").contains(includeAuth)) {
+ throw new IOException(
+ String.format(
+ CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0,
+ "include-auth"));
+ }
+ report.put("authorizationObjects", authorizationObjects);
+ report.put("authorizationRestoreSelected", Boolean.parseBoolean(includeAuth));
+ if (authorizationObjects) {
+ report.put(
+ "authorizationSensitiveFields",
+ List.of("password-hashes", "role-memberships", "privileges"));
+ }
+ report.put("conflict", "fail");
+ report.put(
+ "requestCount",
+ streams.stream()
+ .flatMap(stream -> stream.getEventGroups().stream())
+ .mapToLong(group -> group.getRequests().size())
+ .sum());
+ report.put("dryRun", line.hasOption(OPTION_DRY_RUN));
+ final boolean targetProvided =
+ line.hasOption(OPTION_HOST)
+ || line.hasOption(OPTION_PORT)
+ || line.hasOption(OPTION_USER)
+ || line.hasOption(OPTION_PASSWORD_STDIN)
+ || line.hasOption(OPTION_PASSWORD_ENV);
+ if (!targetProvided && line.hasOption(OPTION_DRY_RUN)) {
+ report.put("targetPreflight", "not-requested");
+ printSummary(line, report);
return 0;
}
+ if (authorizationObjects
+ && !Boolean.parseBoolean(includeAuth)
+ && !line.hasOption(OPTION_DRY_RUN)) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79);
+ }
final String host = required(line, OPTION_HOST);
final int port = Integer.parseInt(required(line, OPTION_PORT));
final String user = line.getOptionValue(OPTION_USER, "root");
final String password = password(line);
+ report.put("target", Map.of("host", host, "port", port, "user", user));
final Path checkpoint = checkpointPath(line);
- final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user);
+ final boolean resume = line.hasOption(OPTION_RESUME);
+ if (!line.hasOption(OPTION_DRY_RUN)
+ && Files.exists(checkpoint, LinkOption.NOFOLLOW_LINKS) != resume) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D);
+ }
+ final ImportCheckpoint state =
+ line.hasOption(OPTION_DRY_RUN)
+ ? null
+ : readCheckpoint(checkpoint, streams, host, port, user);
+ if (state != null) {
+ final String sourceDigest = sourceDigest(Path.of(input(line)), streams);
+ if (state.sourceSha256 != null && !state.sourceSha256.equals(sourceDigest)) {
+ throw invalidCheckpoint(checkpoint);
+ }
+ state.sourceSha256 = sourceDigest;
+ }
+ try (final Session session =
+ new Session.Builder().host(host).port(port).username(user).password(password).build()) {
+ session.open();
+ final String precision = session.getTimestampPrecision();
+ if (!streams.get(0).getManifest().timestampPrecision.equals(precision)) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB);
+ }
+ String version = "unknown";
+ try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW VERSION")) {
+ final SessionDataSet.DataIterator iterator = dataSet.iterator();
+ if (iterator.next()) {
+ version = iterator.getString(dataSet.getColumnNames().get(0));
+ }
+ }
+ final LogicalBackupCatalog catalog = streams.get(0).getCatalog();
+ final String sourceVersion = streams.get(0).getManifest().sourceVersion;
+ if ((sourceVersion != null && !version.split("\\.")[0].equals(sourceVersion.split("\\.")[0]))
+ || (catalog != null && catalog.model.contains("table") && version.startsWith("1."))) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C);
+ }
+ report.put("targetVersion", version);
+ report.put("timestampPrecision", precision);
+ }
try (final IoTDBSyncClient client =
new IoTDBSyncClient(
new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(),
@@ -537,32 +908,87 @@ private static int importBackup(final CommandLine line) throws Exception {
false,
"",
"")) {
- handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
- if (containsConfigStream(streams)) {
- configNodeHandshake(
- client, streams.get(0).getManifest().timestampPrecision, user, password);
+ preflight(client, streams, user, password);
+ report.put("targetPreflight", "passed");
+ report.put("conflictCheck", "on-replay");
+ if (line.hasOption(OPTION_DRY_RUN)) {
+ printSummary(line, report);
+ return 0;
}
- long importedGroups = 0;
- for (final BackupStream stream : streams) {
- importedGroups +=
- importStream(
- checkpoint,
- checkpointState,
- stream,
- request -> transfer(client, request),
- state -> writeCheckpoint(checkpoint, state));
+ (line.hasOption(OPTION_JSON) ? System.err : System.out).println(GSON.toJson(report));
+ final java.util.concurrent.atomic.AtomicBoolean uncertainRequest =
+ new java.util.concurrent.atomic.AtomicBoolean(
+ state.inProgress != null
+ && state.inProgress.uncertain
+ && streams.stream()
+ .flatMap(stream -> stream.getEventGroups().stream())
+ .anyMatch(
+ group ->
+ group
+ .getEventGroupId()
+ .toString()
+ .equals(state.inProgress.eventGroupId)
+ && state.inProgress.nextRequestIndex
+ < group.getRequests().size()));
+ try {
+ state.status = "RUNNING";
+ state.lastError = null;
+ long importedGroups = 0;
+ for (final BackupStream stream : streams) {
+ importedGroups +=
+ importStream(
+ checkpoint,
+ state,
+ stream,
+ request -> transfer(client, request, uncertainRequest.getAndSet(false)),
+ checkpointState -> writeCheckpoint(checkpoint, checkpointState));
+ }
+ state.status = "COMPLETED";
+ writeCheckpoint(checkpoint, state);
+ report.put("status", state.status);
+ report.put("importedEventGroups", importedGroups);
+ report.put("checkpoint", checkpoint.toString());
+ printSummary(line, report);
+ } catch (final Exception e) {
+ state.status = "FAILED";
+ state.lastError = e.getClass().getSimpleName();
+ try {
+ writeCheckpoint(checkpoint, state);
+ } catch (final IOException checkpointFailure) {
+ e.addSuppressed(checkpointFailure);
+ }
+ throw e;
}
- writeCheckpoint(checkpoint, checkpointState);
- System.out.println(
- String.format(
- CliMessages
- .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D,
- importedGroups,
- checkpoint));
}
return 0;
}
+ static void preflight(
+ final IoTDBSyncClient client,
+ final List streams,
+ final String user,
+ final String password)
+ throws IOException, TException {
+ handshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
+ if (containsConfigStream(streams)) {
+ configNodeHandshake(client, streams.get(0).getManifest().timestampPrecision, user, password);
+ }
+ }
+
+ private static String sourceDigest(final Path input, final List streams)
+ throws IOException {
+ if (Files.isRegularFile(input)
+ && !LogicalBackupFormat.MANIFEST_FILE_NAME.equals(input.getFileName().toString())) {
+ return LogicalBackupCatalog.sha256(input);
+ }
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ updateDigest(digest, GSON.toJson(sourceStreamIdentities(streams)));
+ if (!streams.isEmpty() && streams.get(0).getCatalog() != null) {
+ updateDigest(digest, GSON.toJson(streams.get(0).getCatalog()));
+ }
+ return LogicalBackupFormat.toHex(digest.digest());
+ }
+
static boolean containsConfigStream(final List streams) {
return streams.stream()
.anyMatch(stream -> "config".equalsIgnoreCase(stream.getManifest().streamType));
@@ -598,8 +1024,16 @@ static long importStream(
throw invalidCheckpoint(checkpoint);
}
for (int index = nextRequestIndex; index < requests.size(); index++) {
- requestTransfer.transfer(requests.get(index).toTPipeTransferReq());
+ checkpointState.inProgress.uncertain = true;
+ checkpointPersister.persist(checkpointState);
+ try {
+ requestTransfer.transfer(requests.get(index).toTPipeTransferReq());
+ } catch (final RestoreRequestRejectedException e) {
+ checkpointState.inProgress.uncertain = false;
+ throw e;
+ }
checkpointState.inProgress.nextRequestIndex = index + 1;
+ checkpointState.inProgress.uncertain = false;
checkpointPersister.persist(checkpointState);
}
@@ -737,16 +1171,20 @@ private static void configNodeHandshake(
}
}
- static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request)
+ static void transfer(
+ final IoTDBSyncClient client,
+ final TPipeTransferReq request,
+ final boolean allowIdempotentReplay)
throws TException, IOException {
final TPipeTransferResp response = client.pipeTransfer(request);
if (response == null
|| response.getStatus() == null
|| (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
&& response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
- && response.getStatus().getCode()
- != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) {
- throw new IOException(
+ && (!allowIdempotentReplay
+ || response.getStatus().getCode()
+ != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))) {
+ throw new RestoreRequestRejectedException(
String.format(
CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11,
request.getType(),
@@ -755,12 +1193,29 @@ static void transfer(final IoTDBSyncClient client, final TPipeTransferReq reques
}
private static Path checkpointPath(final CommandLine line) throws IOException {
- final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT);
- if (configuredCheckpoint != null) {
- return Path.of(configuredCheckpoint);
- }
+ final String configuredCheckpoint =
+ line.hasOption(OPTION_CHECKPOINT)
+ ? line.getOptionValue(OPTION_CHECKPOINT)
+ : line.getOptionValue(OPTION_RESUME);
final Path source = Path.of(input(line)).toAbsolutePath().normalize();
- return source.resolveSibling(source.getFileName() + ".import.checkpoint.json");
+ final Path checkpoint =
+ (configuredCheckpoint == null
+ ? source.resolveSibling(source.getFileName() + ".restore.checkpoint.json")
+ : Path.of(configuredCheckpoint))
+ .toAbsolutePath()
+ .normalize();
+ LogicalBackupCatalog.rejectSymbolicLinks(checkpoint);
+ final Path sourceRoot =
+ Files.isDirectory(source)
+ ? source
+ : LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())
+ ? source.getParent()
+ : source;
+ if (checkpoint.startsWith(sourceRoot)) {
+ throw new IOException(
+ CliMessages.EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59);
+ }
+ return checkpoint;
}
static ImportCheckpoint readCheckpoint(
@@ -776,6 +1231,8 @@ static ImportCheckpoint readCheckpoint(
state.formatName = CHECKPOINT_FORMAT_NAME;
state.formatVersion = CHECKPOINT_FORMAT_VERSION;
state.sourceStreams = sourceStreams;
+ state.backupId = streams.get(0).getManifest().backupId;
+ state.backupFormatVersion = streams.get(0).getManifest().formatVersion;
state.targetHost = host;
state.targetPort = port;
state.targetUser = user;
@@ -801,7 +1258,9 @@ static ImportCheckpoint readCheckpoint(
String.format(
CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint));
}
- if (!sourceStreams.equals(state.sourceStreams)
+ if (!streams.get(0).getManifest().backupId.equals(state.backupId)
+ || !streams.get(0).getManifest().formatVersion.equals(state.backupFormatVersion)
+ || !sourceStreams.equals(state.sourceStreams)
|| !host.equals(state.targetHost)
|| port != state.targetPort
|| !user.equals(state.targetUser)
@@ -820,6 +1279,11 @@ static Map sourceStreamIdentities(final List strea
for (final BackupStream stream : streams) {
final LogicalBackupManifest manifest = stream.getManifest();
final MessageDigest digest = LogicalBackupFormat.newSha256();
+ updateDigest(digest, manifest.formatName);
+ updateDigest(digest, manifest.formatVersion);
+ updateDigest(digest, manifest.timestampPrecision);
+ updateDigest(digest, manifest.sourceVersion);
+ updateDigest(digest, GSON.toJson(manifest.captureOptions));
updateDigest(digest, manifest.backupId);
updateDigest(digest, manifest.streamId);
updateDigest(digest, manifest.pipeName);
@@ -946,12 +1410,13 @@ static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkp
if (parent != null) {
Files.createDirectories(parent);
}
- final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp");
- Files.deleteIfExists(temporary);
+ LogicalBackupCatalog.rejectSymbolicLinks(checkpoint);
+ checkpointState.updatedAt = java.time.Instant.now().toString();
+ final Path temporary = Files.createTempFile(parent, ".checkpoint-", ".tmp");
final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState));
try (final java.nio.channels.FileChannel checkpointChannel =
java.nio.channels.FileChannel.open(
- temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ temporary, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
while (checkpointBytes.hasRemaining()) {
checkpointChannel.write(checkpointBytes);
}
@@ -975,47 +1440,20 @@ private static String getClusterId() {
}
private static void printUsage() {
- final HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(
- new PrintWriter(System.out),
- 120,
- CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE,
+ System.out.println(
CliMessages
- .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB,
- optionsForHelp(),
- 2,
- 2,
- "",
- true);
- }
-
- private static Options optionsForHelp() {
- final Options options = new Options();
- addOption(options, OPTION_INPUT, true, false);
- addOption(options, OPTION_OUTPUT, true, false);
- addOption(options, OPTION_RESUME, true, false);
- addOption(options, OPTION_FORMAT, true, false);
- addOption(options, OPTION_DEEP, false, false);
- addOption(options, OPTION_HOST, true, false);
- addOption(options, OPTION_PORT, true, false);
- addOption(options, OPTION_USER, true, false);
- addOption(options, OPTION_PASSWORD_STDIN, false, false);
- addOption(options, OPTION_PASSWORD_ENV, true, false);
- addOption(options, OPTION_CHECKPOINT, true, false);
- addOption(options, OPTION_DRY_RUN, false, false);
- addOption(options, OPTION_ALLOW_INCOMPLETE, false, false);
- addOption(options, OPTION_JSON, false, false);
- addOption(options, OPTION_BACKUP_DIR, true, false);
- addOption(options, OPTION_PIPE_NAME, true, false);
- addOption(options, OPTION_BACKUP_ID, true, false);
- addOption(options, OPTION_TIMEOUT_SECONDS, true, false);
- addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false);
- return options;
+ .LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8);
}
static class ImportCheckpoint {
String formatName;
String formatVersion;
+ String sourceSha256;
+ String backupId;
+ String backupFormatVersion;
+ String status = "RUNNING";
+ String updatedAt;
+ String lastError;
Map sourceStreams = new LinkedHashMap<>();
String targetHost;
int targetPort;
@@ -1029,6 +1467,7 @@ static class ImportProgress {
String eventGroupId;
long lastSequence;
int nextRequestIndex;
+ boolean uncertain;
static ImportProgress start(final String streamId, final EventGroup group) {
final ImportProgress progress = new ImportProgress();
@@ -1045,6 +1484,12 @@ boolean matches(final String candidateStreamId, final EventGroup group) {
}
}
+ private static class RestoreRequestRejectedException extends IOException {
+ private RestoreRequestRejectedException(final String message) {
+ super(message);
+ }
+ }
+
@FunctionalInterface
interface RequestTransfer {
void transfer(TPipeTransferReq request) throws Exception;
@@ -1076,7 +1521,11 @@ private static BackupInput open(final Path source) throws IOException {
return new BackupInput(temporaryDirectory, temporaryDirectory);
} catch (final IOException | RuntimeException e) {
deleteRecursively(temporaryDirectory);
- throw e;
+ if (e instanceof IOException) {
+ throw (IOException) e;
+ }
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e);
}
}
@@ -1084,46 +1533,150 @@ private static void extract(final Path archive, final Path target) throws IOExce
int entryCount = 0;
long totalBytes = 0;
final byte[] buffer = new byte[64 * 1024];
- try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) {
- ZipEntry entry;
- while ((entry = zip.getNextEntry()) != null) {
+ try (final ZipFile zip = new ZipFile(archive.toFile())) {
+ final Map entries = new LinkedHashMap<>();
+ final Set allowed = new LinkedHashSet<>();
+ final java.util.Enumeration enumeration = zip.entries();
+ while (enumeration.hasMoreElements()) {
+ final ZipEntry entry = enumeration.nextElement();
if (++entryCount > MAX_ARCHIVE_ENTRIES) {
throw new IOException(
CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
}
- final Path output = target.resolve(entry.getName()).normalize();
- if (!output.startsWith(target)) {
+ LogicalBackupCatalog.resolve(
+ target,
+ entry.isDirectory()
+ ? entry.getName().substring(0, entry.getName().length() - 1)
+ : entry.getName());
+ if (entries.putIfAbsent(entry.getName(), entry) != null
+ || entry.getSize() < 0
+ || entry.getSize() > MAX_ARCHIVE_BYTES
+ || (entry.getSize() > 1024 * 1024
+ && entry.getSize() / Math.max(1, entry.getCompressedSize()) > 1000)) {
throw new IOException(
- String.format(
- CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152,
- entry.getName()));
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
}
- if (entry.isDirectory()) {
- Files.createDirectories(output);
- } else {
- final Path parent = output.getParent();
- if (parent != null) {
- Files.createDirectories(parent);
+ }
+ final ZipEntry rootManifest = entries.get(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ if (rootManifest != null) {
+ final com.google.gson.JsonObject rootJson = manifestJson(zip, rootManifest);
+ if (rootJson.has("streams")) {
+ final LogicalBackupCatalog catalog =
+ GSON.fromJson(rootJson, LogicalBackupCatalog.class);
+ allowed.add(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ if (catalog.streams == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
}
- try (final OutputStream stream =
- Files.newOutputStream(
- output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
- int length;
- while ((length = zip.read(buffer)) >= 0) {
- if (length == 0) {
- continue;
+ for (final LogicalBackupCatalog.StreamEntry stream : catalog.streams) {
+ if (stream == null
+ || stream.manifest == null
+ || stream.segments == null
+ || !allowed.add(stream.manifest.file)) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ for (final LogicalBackupCatalog.FileEntry segment : stream.segments) {
+ if (segment == null || !allowed.add(segment.file)) {
+ throw new IOException(
+ CliMessages
+ .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
}
- totalBytes += length;
- if (totalBytes > MAX_ARCHIVE_BYTES) {
+ }
+ }
+ }
+ }
+ if (allowed.isEmpty()) {
+ // Read archives produced by the initial stream-only format with the same allowlist.
+ for (final ZipEntry entry : entries.values()) {
+ if (!entry.isDirectory() && entry.getName().endsWith("manifest.json")) {
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(manifestJson(zip, entry), LogicalBackupManifest.class);
+ if (manifest.segments == null || !allowed.add(entry.getName())) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ final Path parent = Path.of(entry.getName()).getParent();
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ if (segment == null
+ || segment.file == null
+ || !allowed.add(
+ parent == null
+ ? segment.file
+ : parent.resolve(segment.file).toString().replace('\\', '/'))) {
throw new IOException(
- CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ CliMessages
+ .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
}
- stream.write(buffer, 0, length);
}
}
}
- zip.closeEntry();
}
+ if (allowed.isEmpty()) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ for (final String file : allowed) {
+ LogicalBackupCatalog.resolve(target, file);
+ if (!entries.containsKey(file) || entries.get(file).isDirectory()) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ }
+ for (final ZipEntry entry : entries.values()) {
+ if (entry.isDirectory()) {
+ if (allowed.stream().noneMatch(file -> file.startsWith(entry.getName()))) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ continue;
+ }
+ if (!allowed.contains(entry.getName())) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ final Path output = LogicalBackupCatalog.resolve(target, entry.getName());
+ Files.createDirectories(output.getParent());
+ try (final java.io.InputStream input = zip.getInputStream(entry);
+ final OutputStream stream =
+ Files.newOutputStream(
+ output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+ int length;
+ while ((length = input.read(buffer)) >= 0) {
+ if (length == 0) {
+ continue;
+ }
+ totalBytes += length;
+ if (totalBytes > MAX_ARCHIVE_BYTES) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ stream.write(buffer, 0, length);
+ }
+ }
+ }
+ }
+ }
+
+ private static com.google.gson.JsonObject manifestJson(final ZipFile zip, final ZipEntry entry)
+ throws IOException {
+ if (entry.isDirectory() || entry.getSize() > 16 * 1024 * 1024) {
+ throw new IOException(
+ CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432);
+ }
+ try (final java.io.InputStream input = zip.getInputStream(entry)) {
+ final byte[] bytes = input.readNBytes(16 * 1024 * 1024 + 1);
+ final com.google.gson.JsonObject json =
+ GSON.fromJson(
+ new String(bytes, StandardCharsets.UTF_8), com.google.gson.JsonObject.class);
+ if (bytes.length > 16 * 1024 * 1024 || json == null) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81);
+ }
+ return json;
+ } catch (final JsonParseException | IllegalStateException e) {
+ throw new IOException(
+ CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e);
}
}
diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
index f39b87f0754e..4fee78c4b15b 100644
--- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
+++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java
@@ -23,9 +23,11 @@
import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
@@ -37,8 +39,11 @@
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -57,7 +62,7 @@ public class PipeLogicalBackupToolTest {
@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
- public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
+ public void testDirectoryPackZipVerifyAndRestoreDryRun() throws Exception {
final Path root = temporaryFolder.getRoot().toPath();
final Path source = root.resolve("source");
writeBackup(source);
@@ -70,20 +75,14 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
Assert.assertEquals(
0,
PipeLogicalBackupTool.run(
- new String[] {
- "export",
- "--input",
- source.toString(),
- "--output",
- archive.toString(),
- "--format=binary"
- }));
+ new String[] {"pack", "--input", source.toString(), "--output", archive.toString()}));
Assert.assertTrue(Files.isRegularFile(archive));
try (final ZipFile zip = new ZipFile(archive.toFile())) {
Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME));
- Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal"));
+ Assert.assertNotNull(zip.getEntry("legacy-stream/manifest.json"));
+ Assert.assertNotNull(zip.getEntry("legacy-stream/segment-00000000000000000000.pwal"));
Assert.assertNull(zip.getEntry(".backup.lock"));
- Assert.assertEquals(2, zip.size());
+ Assert.assertEquals(3, zip.size());
}
Assert.assertEquals(
@@ -91,7 +90,7 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
Assert.assertEquals(
0,
PipeLogicalBackupTool.run(
- new String[] {"import", "--input", archive.toString(), "--dry-run"}));
+ new String[] {"restore", "--input", archive.toString(), "--dry-run"}));
final byte[] originalArchive = Files.readAllBytes(archive);
try {
@@ -103,6 +102,61 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception {
}
}
+ @Test
+ public void testPackPreservesIncompleteStatus() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("incomplete");
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup";
+ manifest.pipeName = "pipe";
+ manifest.pipeCreationTime = 1;
+ manifest.streamId = "data-1";
+ manifest.streamType = "data";
+ manifest.timestampPrecision = "ms";
+ try (final LogicalBackupWriter writer =
+ new LogicalBackupWriter(
+ source, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) {
+ writer.recordSkippedEvent(1, "unsupported event");
+ }
+ final Path archive = root.resolve("incomplete.zip");
+ try {
+ PipeLogicalBackupTool.run(
+ new String[] {"pack", "--input", source.toString(), "--output", archive.toString()});
+ Assert.fail();
+ } catch (final IOException expected) {
+ Assert.assertFalse(Files.exists(archive));
+ }
+ final PrintStream previous = System.out;
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try {
+ System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8));
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "pack",
+ "--input",
+ source.toString(),
+ "--output",
+ archive.toString(),
+ "--allow-incomplete",
+ "--json"
+ });
+ final com.google.gson.JsonObject json =
+ new com.google.gson.Gson()
+ .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class);
+ Assert.assertEquals("INCOMPLETE", json.get("status").getAsString());
+ bytes.reset();
+ PipeLogicalBackupTool.run(
+ new String[] {"verify", "--input", archive.toString(), "--allow-incomplete", "--json"});
+ final com.google.gson.JsonObject verified =
+ new com.google.gson.Gson()
+ .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class);
+ Assert.assertEquals(
+ "INCOMPLETE", verified.getAsJsonObject("manifest").get("status").getAsString());
+ } finally {
+ System.setOut(previous);
+ }
+ }
+
@Test
public void testCheckpointSourceIdentityIncludesContent() throws Exception {
final Path source = temporaryFolder.getRoot().toPath().resolve("source");
@@ -166,11 +220,11 @@ public void testPlaintextPasswordOptionIsRejected() throws Exception {
}
@Test
- public void testRemovedCommandAliasesAreRejected() throws Exception {
+ public void testUnknownCommandsAreRejected() throws Exception {
final Path source = temporaryFolder.getRoot().toPath().resolve("source");
writeBackup(source);
- for (final String command : Arrays.asList("restore", "stats")) {
+ for (final String command : Arrays.asList("stats", "generate")) {
try {
PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()});
Assert.fail();
@@ -180,6 +234,180 @@ public void testRemovedCommandAliasesAreRejected() throws Exception {
}
}
+ @Test
+ public void testRootManifestPackAndJsonVerification() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("backup");
+ writeBackup(source.resolve("data-1"));
+ final LogicalBackupCatalog catalog = new LogicalBackupCatalog();
+ catalog.backupId = "backup";
+ catalog.pipeName = "pipe";
+ catalog.scope = Collections.singletonList("data");
+ catalog.model = Collections.singletonList("tree");
+ catalog.seal(source, new LogicalBackupArchiveReader().read(source, false));
+ final byte[] originalManifest = Files.readAllBytes(source.resolve("manifest.json"));
+ final Path archive = root.resolve("rooted.zip");
+ PipeLogicalBackupTool.run(
+ new String[] {"pack", "--input", source.toString(), "--output", archive.toString()});
+ Assert.assertArrayEquals(originalManifest, Files.readAllBytes(source.resolve("manifest.json")));
+ try (final ZipFile zip = new ZipFile(archive.toFile())) {
+ Assert.assertNotNull(zip.getEntry("manifest.json"));
+ Assert.assertNotNull(zip.getEntry("data-1/manifest.json"));
+ Assert.assertEquals(3, zip.size());
+ }
+ final PrintStream previous = System.out;
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try {
+ System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8));
+ PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString(), "--json"});
+ final com.google.gson.JsonObject json =
+ new com.google.gson.Gson()
+ .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class);
+ Assert.assertEquals("passed", json.get("checksum").getAsString());
+ Assert.assertEquals("backup", json.getAsJsonObject("manifest").get("backupId").getAsString());
+ } finally {
+ System.setOut(previous);
+ }
+ }
+
+ @Test
+ public void testDeprecatedAliasesReportMigration() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath();
+ final Path source = root.resolve("source");
+ writeBackup(source);
+ final PrintStream previous = System.err;
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try {
+ System.setErr(new PrintStream(bytes, true, StandardCharsets.UTF_8));
+ PipeLogicalBackupTool.run(
+ new String[] {
+ "export",
+ "--input",
+ source.toString(),
+ "--output",
+ root.resolve("legacy.zip").toString()
+ });
+ PipeLogicalBackupTool.run(new String[] {"import", "--input", source.toString(), "--dry-run"});
+ } finally {
+ System.setErr(previous);
+ }
+ final String notices = bytes.toString(StandardCharsets.UTF_8);
+ Assert.assertTrue(notices.contains("dump"));
+ Assert.assertTrue(notices.contains("pack"));
+ Assert.assertTrue(notices.contains("restore"));
+ }
+
+ @Test
+ public void testCommandsRejectOptionsWithOtherResponsibilities() throws Exception {
+ final List invalid =
+ Arrays.asList(
+ new String[] {
+ "pack", "--input", "backup", "--output", "backup.zip", "--host", "localhost"
+ },
+ new String[] {"dump", "--input", "backup"},
+ new String[] {"dump", "--output", "backup.zip"},
+ new String[] {"inspect", "--input", "backup", "--password-stdin"},
+ new String[] {"restore", "--input", "backup", "--resume", "checkpoint.json"},
+ new String[] {"restore", "--input", "backup", "--conflict", "overwrite"});
+ for (final String[] args : invalid) {
+ try {
+ PipeLogicalBackupTool.run(args);
+ Assert.fail(Arrays.toString(args));
+ } catch (final ParseException expected) {
+ // Command-specific validation must fail before opening files or connections.
+ }
+ }
+ }
+
+ @Test
+ public void testDumpScopeAuthAndWriterOptionsAreExplicit() throws Exception {
+ final LogicalBackupDumpOptions options =
+ new LogicalBackupDumpOptions(
+ PipeLogicalBackupTool.parse(
+ "dump",
+ new String[] {
+ "--scope",
+ "schema,data",
+ "--model",
+ "tree",
+ "--include",
+ "root.a.**",
+ "--fsync-policy",
+ "periodic",
+ "--fsync-period-ms",
+ "42"
+ }),
+ false);
+ Assert.assertEquals("schema,data", options.source.get("source.inclusion"));
+ Assert.assertEquals("auth", options.source.get("source.inclusion.exclusion"));
+ Assert.assertEquals("false", options.source.get("source.capture.table"));
+ Assert.assertEquals("root.a.**", options.source.get("source.path.inclusion"));
+ Assert.assertEquals("42", options.sink.get("sink.fsync-period-ms"));
+ Assert.assertFalse(options.catalog.includeAuth);
+ final LogicalBackupDumpOptions auth =
+ new LogicalBackupDumpOptions(
+ PipeLogicalBackupTool.parse(
+ "dump", new String[] {"--scope", "config", "--include-auth", "true"}),
+ false);
+ Assert.assertTrue(auth.source.get("source.inclusion").contains("auth"));
+ }
+
+ @Test
+ public void testDumpRejectsAmbiguousOrUnboundedCaptureBeforeCreatingTask() throws Exception {
+ for (final String[] args :
+ Arrays.asList(
+ new String[] {},
+ new String[] {"--scope", "data", "--mode", "continuous"},
+ new String[] {"--scope", "data", "--include-auth", "true"},
+ new String[] {"--scope", "data", "--model", "tree,table", "--include", "root.a.**"},
+ new String[] {"--scope", "data", "--segment-size-bytes", "1024"},
+ new String[] {"--scope", "data", "--fsync-period-ms", "1000"},
+ new String[] {"--scope", "data", "--mode", "continuous", "--stop-sequence", "100"})) {
+ try {
+ new LogicalBackupDumpOptions(PipeLogicalBackupTool.parse("dump", args), false);
+ Assert.fail(Arrays.toString(args));
+ } catch (final IOException expected) {
+ // No unsupported option may broaden capture or leave an unbounded task.
+ }
+ }
+ }
+
+ @Test
+ public void testPreflightSendsOnlyHandshakeRequests() throws Exception {
+ final Path source = temporaryFolder.getRoot().toPath().resolve("source");
+ writeBackup(source, "config-0", "config", Collections.singletonList(request((short) 200)));
+ final List streams = new LogicalBackupArchiveReader().read(source, false);
+ final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class);
+ Mockito.when(client.pipeTransfer(Mockito.any()))
+ .thenReturn(
+ new TPipeTransferResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())));
+ PipeLogicalBackupTool.preflight(client, streams, "root", "secret");
+ final org.mockito.ArgumentCaptor requests =
+ org.mockito.ArgumentCaptor.forClass(TPipeTransferReq.class);
+ Mockito.verify(client, Mockito.times(2)).pipeTransfer(requests.capture());
+ Assert.assertEquals(
+ PipeRequestType.HANDSHAKE_DATANODE_V2.getType(), requests.getAllValues().get(0).getType());
+ Assert.assertEquals(
+ PipeRequestType.HANDSHAKE_CONFIGNODE_V2.getType(),
+ requests.getAllValues().get(1).getType());
+ }
+
+ @Test
+ public void testFreshRestoreRejectsIdempotentConflict() throws Exception {
+ final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class);
+ Mockito.when(client.pipeTransfer(Mockito.any()))
+ .thenReturn(
+ new TPipeTransferResp(
+ new TSStatus(
+ TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())));
+ try {
+ PipeLogicalBackupTool.transfer(client, request(1), false);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // Only the uncertain request at a durable resume point may accept an idempotent conflict.
+ }
+ }
+
@Test
public void testTransferAcceptsIdempotentConflict() throws Exception {
final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class);
@@ -190,7 +418,7 @@ public void testTransferAcceptsIdempotentConflict() throws Exception {
new TSStatus(
TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())));
- PipeLogicalBackupTool.transfer(client, request);
+ PipeLogicalBackupTool.transfer(client, request, true);
}
@Test
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
index f83749348115..fe946004e3be 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
@@ -22,6 +22,7 @@
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException;
import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
@@ -32,13 +33,18 @@
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
import org.apache.iotdb.commons.pipe.resource.log.PipeLogger;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager;
+import org.apache.iotdb.confignode.manager.pipe.source.ConfigRegionListeningFilter;
import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter;
+import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -185,12 +191,21 @@ private void parseHeartbeatAndSaveMetaChangeLocally(
// pipes must remain alive because they may receive work in the future.
final boolean isFiniteInternalPipe =
!staticMeta.isSourceExternal()
- && PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters());
+ && (PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters())
+ || (staticMeta
+ .getSourceParameters()
+ .getLongOrDefault("source.logical-backup.until", -1L)
+ > 0
+ && System.currentTimeMillis()
+ >= staticMeta
+ .getSourceParameters()
+ .getLongOrDefault("source.logical-backup.until", -1L)));
final boolean hasReliableDataRegionReport =
pipeHeartbeat.hasCompletedDataRegionReport(staticMeta);
if (isFiniteInternalPipe
&& hasReliableDataRegionReport
- && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)) {
+ && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)
+ && metadataSnapshotsCompleted(pipeMetaFromCoordinator)) {
PipeLogger.log(
LOGGER::info,
ManagerMessages.ALL_DATANODES_REPORTED_HISTORICAL_PIPE_COMPLETED,
@@ -329,6 +344,54 @@ private Set collectRequiredDataRegionIds(final PipeMeta pipeMetaFromCoo
return requiredDataRegionIds;
}
+ private boolean metadataSnapshotsCompleted(final PipeMeta pipeMeta) {
+ final PipeParameters parameters = pipeMeta.getStaticMeta().getSourceParameters();
+ if (!parameters.getBooleanOrDefault("source.logical-backup.snapshot", false)
+ && parameters.getLongOrDefault("source.logical-backup.until", -1L) <= 0) {
+ return true;
+ }
+ try {
+ for (final Map.Entry entry :
+ pipeMeta.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().entrySet()) {
+ final boolean required;
+ if (entry.getKey() == Integer.MIN_VALUE) {
+ required = !ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters).isEmpty();
+ } else {
+ final TConsensusGroupId region =
+ new TConsensusGroupId(TConsensusGroupType.SchemaRegion, entry.getKey());
+ if (!configManager.getPartitionManager().isRegionGroupExists(region)
+ || !SchemaRegionListeningFilter.shouldSchemaRegionBeListened(parameters)) {
+ continue;
+ }
+ final String database = configManager.getPartitionManager().getRegionDatabase(region);
+ final TDatabaseSchema schema =
+ configManager.getClusterSchemaManager().getDatabaseSchemaByName(database);
+ required =
+ schema.isIsTableModel()
+ ? TablePattern.isTableModelDataAllowToBeCaptured(parameters)
+ : TreePattern.isTreeModelDataAllowToBeCaptured(parameters);
+ }
+ if (required
+ && entry
+ .getValue()
+ .getProgressIndex()
+ .getProgressIndexByType(MetaProgressIndex.class)
+ .map(
+ index ->
+ index.getIndex()
+ != org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource
+ .LOGICAL_BACKUP_COMPLETED_INDEX)
+ .orElse(true)) {
+ return false;
+ }
+ }
+ return true;
+ } catch (final Exception e) {
+ // A failed metadata lookup must never turn a partial snapshot into a completed backup.
+ return false;
+ }
+ }
+
private boolean shouldKeepDataRegionAsRequired(
final PipeStaticMeta staticMeta, final TConsensusGroupId dataRegionId) {
if (staticMeta.isSourceExternal()) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
index 8c7f28f6c714..789953a05d4d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -8,12 +8,24 @@
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionSnapshotEvent;
import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionWritePlanEvent;
@@ -21,6 +33,8 @@
import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotPieceReq;
import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotSealReq;
import org.apache.iotdb.confignode.service.ConfigNode;
+import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.pipe.api.PipeConnector;
import org.apache.iotdb.pipe.api.annotation.TableModel;
import org.apache.iotdb.pipe.api.annotation.TreeModel;
@@ -34,8 +48,6 @@
import java.io.File;
import java.io.FileInputStream;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -47,11 +59,15 @@
public class LogicalBackupSink implements PipeConnector {
private static final int FILE_PIECE_BYTES = 64 * 1024;
private LogicalBackupWriter writer;
+ private LogicalBackupSettings settings;
@Override
public void validate(final PipeParameterValidator validator) throws Exception {
validator.validateSynonymAttributes(
Collections.singletonList("connector.dir"), Collections.singletonList("sink.dir"), true);
+ final LogicalBackupSettings validated =
+ new LogicalBackupSettings(validator.getParameters(), "backup");
+ validated.requireSharedResume(validator.getParameters());
}
@Override
@@ -61,18 +77,13 @@ public void customize(
final String pipeName = configuration.getRuntimeEnvironment().getPipeName();
final long creationTime = configuration.getRuntimeEnvironment().getCreationTime();
final int regionId = configuration.getRuntimeEnvironment().getRegionId();
- final String root = parameters.getStringByKeys("connector.dir", "sink.dir");
- final String backupId =
- parameters.getStringOrDefault(
- Arrays.asList("connector.backup-id", "sink.backup-id"), pipeName + "-" + creationTime);
- final Path directory =
- Paths.get(root)
- .toAbsolutePath()
- .normalize()
- .resolve(backupId)
- .resolve("config-" + regionId);
+ settings = new LogicalBackupSettings(parameters, pipeName + "-" + creationTime);
+ settings.requireSharedResume(parameters);
+ if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) {
+ return;
+ }
final LogicalBackupManifest manifest = new LogicalBackupManifest();
- manifest.backupId = backupId;
+ manifest.backupId = settings.backupId;
manifest.pipeName = pipeName;
manifest.pipeCreationTime = creationTime;
manifest.sourceClusterId =
@@ -83,23 +94,18 @@ public void customize(
manifest.streamId = pipeName + "-" + creationTime + "-config-" + regionId;
manifest.regionId = regionId;
manifest.streamType = "config";
- writer =
- new LogicalBackupWriter(
- directory,
- manifest,
- 64 * 1024 * 1024,
- 64 * 1024 * 1024,
- LogicalBackupWriter.FsyncPolicy.BATCH,
- 100,
- 1000,
- true);
+ writer = settings.open("config", regionId, manifest);
}
@Override
- public void transfer(final TabletInsertionEvent event) throws Exception {}
+ public void transfer(final TabletInsertionEvent event) throws Exception {
+ unsupported(event);
+ }
@Override
- public void transfer(final TsFileInsertionEvent event) throws Exception {}
+ public void transfer(final TsFileInsertionEvent event) throws Exception {
+ unsupported(event);
+ }
@Override
public void transfer(final Event event) throws Exception {
@@ -108,7 +114,7 @@ public void transfer(final Event event) throws Exception {
if (e.increaseReferenceCount(getClass().getName())) {
try {
writer.writeEvent(
- UUID.randomUUID(),
+ eventId(e),
System.currentTimeMillis(),
Collections.singletonList(
PipeTransferConfigPlanReq.toTPipeTransferReq(e.getConfigPhysicalPlan())),
@@ -140,13 +146,48 @@ public void transfer(final Event event) throws Exception {
e.getFileType(),
e.toSealTypeString(),
e.getAuthUserName()));
- writer.writeEvent(
- UUID.randomUUID(), System.currentTimeMillis(), requests, "config-snapshot");
+ writer.writeEvent(eventId(e), System.currentTimeMillis(), requests, "config-snapshot");
} finally {
e.decreaseReferenceCount(getClass().getName(), false);
}
}
+ } else if (event instanceof PipeHeartbeatEvent) {
+ if (settings.includeHeartbeat) {
+ writer.writeControl(
+ LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), "heartbeat");
+ }
+ } else if (event instanceof PipeTerminateEvent) {
+ // close() writes STREAM_END after every preceding request has been committed.
+ } else {
+ unsupported(event);
+ }
+ }
+
+ private void unsupported(final Event event) throws Exception {
+ final String name = event == null ? "null" : event.getClass().getName();
+ if ("skip".equals(settings.unsupported)) {
+ writer.recordSkippedEvent(System.currentTimeMillis(), name);
+ } else {
+ throw new java.io.IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60, name));
+ }
+ }
+
+ private static UUID eventId(final EnrichedEvent event) {
+ String identity =
+ event.getClass().getName()
+ + "\0"
+ + event.getPipeName()
+ + "\0"
+ + event.getCreationTime()
+ + "\0"
+ + event.getCommitId();
+ if (event instanceof PipeConfigRegionSnapshotEvent) {
+ final PipeConfigRegionSnapshotEvent snapshot = (PipeConfigRegionSnapshotEvent) event;
+ identity += "\0" + snapshot.getFileType() + "\0" + snapshot.getSnapshotFile().getName();
}
+ return UUID.nameUUIDFromBytes(identity.getBytes(java.nio.charset.StandardCharsets.UTF_8));
}
private static void appendFile(final List requests, final File file)
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
index b76156bb6192..e066266396e9 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
@@ -22,6 +22,7 @@
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TPipeCompletedDataRegion;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkCriticalException;
@@ -75,6 +76,37 @@ public class PipeHeartbeatParserTest {
private boolean originalSeparatedPipeHeartbeatEnabled;
+ @Test
+ public void testBackupWaitsForCommittedConfigSnapshotAfterDataCompletes() throws Exception {
+ CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false);
+ final Map source = new HashMap<>();
+ source.put("source.realtime.enable", "false");
+ source.put("source.logical-backup.snapshot", "true");
+ source.put("source.inclusion", "schema.database");
+ final PipeMeta meta = createPipeMeta(source, Integer.MIN_VALUE);
+ final PipeTaskInfo info = new PipeTaskInfo();
+ info.createPipe(new CreatePipePlanV2(meta.getStaticMeta(), meta.getRuntimeMeta()));
+ final ParserTestContext context = createParserTestContext(1, info);
+ context.parser.parseHeartbeat(
+ 1,
+ createPipeHeartbeatWithCompletedRegions(
+ meta, true, Collections.singletonList(Integer.MIN_VALUE)));
+ Assert.assertNotNull(info.getPipeMetaByPipeName("test_pipe"));
+ info.getPipeMetaByPipeName("test_pipe")
+ .getRuntimeMeta()
+ .getConsensusGroupId2TaskMetaMap()
+ .get(Integer.MIN_VALUE)
+ .updateProgressIndex(
+ new MetaProgressIndex(
+ org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource
+ .LOGICAL_BACKUP_COMPLETED_INDEX));
+ context.parser.parseHeartbeat(
+ 1,
+ createPipeHeartbeatWithCompletedRegions(
+ meta, true, Collections.singletonList(Integer.MIN_VALUE)));
+ Assert.assertNull(info.getPipeMetaByPipeName("test_pipe"));
+ }
+
@Before
public void setUp() {
originalSeparatedPipeHeartbeatEnabled =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
index 7ac1f7ba12a4..ef404d4e5f8d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java
@@ -81,12 +81,13 @@ public synchronized String register(
connectorKey);
final boolean isDataRegionSink = isDataRegionSink(environment.getRegionId());
+ final boolean isLogicalBackupSink = isLogicalBackupSink(connectorKey);
final int sinkNum = calculateSinkSubtaskNum(pipeSinkParameters, isDataRegionSink, connectorKey);
boolean realTimeFirst = false;
final String attributeSortedString =
generateAttributeSortedString(pipeSinkParameters, environment.getRegionId());
final String attributeDisplayString = generateAttributeDisplayString(pipeSinkParameters);
- if (isDataRegionSink) {
+ if (isDataRegionSink && !isLogicalBackupSink) {
realTimeFirst =
pipeSinkParameters.getBooleanOrDefault(
Arrays.asList(
@@ -95,11 +96,16 @@ public synchronized String register(
PipeSinkConstant.CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE);
}
final String attributeDisplayStringWithPrefix =
- isDataRegionSink
- ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters)
- ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString
- : "data_" + attributeDisplayString
- : "schema_" + attributeDisplayString;
+ isLogicalBackupSink
+ ? (isDataRegionSink ? "data_region_" : "schema_region_")
+ + environment.getRegionId()
+ + "_"
+ + attributeDisplayString
+ : isDataRegionSink
+ ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters)
+ ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString
+ : "data_" + attributeDisplayString
+ : "schema_" + attributeDisplayString;
environment.setAttributeSortedString(attributeDisplayStringWithPrefix);
final PipeSinkSubtaskKey pipeSinkSubtaskKey =
new PipeSinkSubtaskKey(
@@ -386,6 +392,13 @@ public static int calculateSinkSubtaskNum(
public static String generateAttributeSortedString(
final PipeParameters pipeSinkParameters, final int regionId) {
final String attributeSortedString = generateAttributeSortedString(pipeSinkParameters);
+ if (isLogicalBackupSink(getConnectorKey(pipeSinkParameters))) {
+ // A backup stream belongs to exactly one region and must have one ordered writer.
+ return (isDataRegionSink(regionId) ? "data_region_" : "schema_region_")
+ + regionId
+ + "_"
+ + attributeSortedString;
+ }
if (isDataRegionSink(regionId)) {
return PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters)
? "data_region_" + regionId + "_" + attributeSortedString
@@ -398,6 +411,11 @@ private static String getConnectorKey(final PipeParameters pipeSinkParameters) {
return PipeSinkConstant.getConnectorOrSinkNameWithDefault(pipeSinkParameters).toLowerCase();
}
+ private static boolean isLogicalBackupSink(final String connectorKey) {
+ return BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName().equals(connectorKey)
+ || BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName().equals(connectorKey);
+ }
+
private static boolean isDataRegionSink(final int regionId) {
return StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId))
|| PipeRuntimeMeta.isSourceExternal(regionId);
@@ -407,7 +425,7 @@ private static int calculateSinkSubtaskNum(
final PipeParameters pipeSinkParameters,
final boolean isDataRegionSink,
final String connectorKey) {
- if (!isDataRegionSink) {
+ if (!isDataRegionSink || isLogicalBackupSink(connectorKey)) {
// Do not allow parallel tasks for schema region connectors to avoid the potential disorder of
// the schema region data transfer.
return 1;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
index c48238a5ce22..a1ef953f4a00 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java
@@ -22,10 +22,13 @@
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings;
import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter;
+import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
@@ -41,7 +44,11 @@
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq;
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2;
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
import org.apache.iotdb.pipe.api.PipeConnector;
import org.apache.iotdb.pipe.api.annotation.TableModel;
import org.apache.iotdb.pipe.api.annotation.TreeModel;
@@ -55,19 +62,19 @@
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.write.record.Tablet;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
+import java.util.Optional;
import java.util.UUID;
@TreeModel
@@ -82,6 +89,8 @@ public class LogicalBackupSink implements PipeConnector {
protected boolean includeHeartbeat;
protected String unsupportedEventPolicy;
private final String streamType;
+ private long backupStartTime = Long.MIN_VALUE;
+ private long backupEndTime = Long.MAX_VALUE;
public LogicalBackupSink() {
this("unknown");
@@ -161,6 +170,7 @@ public void validate(final PipeParameterValidator validator) throws Exception {
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC,
PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE);
validateSharedDirectoryResumePolicy(parameters);
+ new LogicalBackupSettings(parameters, "backup");
final String policy =
parameters
.getStringOrDefault(
@@ -218,135 +228,27 @@ public void customize(
pipeName = configuration.getRuntimeEnvironment().getPipeName();
creationTime = configuration.getRuntimeEnvironment().getCreationTime();
regionId = configuration.getRuntimeEnvironment().getRegionId();
- final String root =
- parameters.getStringByKeys(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY);
- if (root == null || root.isBlank()) {
- throw new PipeParameterNotValidException(
- DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB);
- }
- final String backupId =
- parameters.getStringOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY),
- pipeName + "-" + creationTime);
- if (backupId == null || backupId.isBlank()) {
- throw new PipeParameterNotValidException(
- String.format(
- DataNodePipeMessages
- .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
- backupId));
- }
- final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId;
- final Path backupRoot = Paths.get(root).toAbsolutePath().normalize();
- final Path backupIdPath;
- try {
- backupIdPath = Path.of(backupId);
- } catch (final RuntimeException e) {
- throw new PipeParameterNotValidException(
- String.format(
- DataNodePipeMessages
- .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
- backupId));
- }
- if (backupIdPath.isAbsolute()
- || backupIdPath.getNameCount() != 1
- || ".".equals(backupId)
- || "..".equals(backupId)) {
- throw new PipeParameterNotValidException(
- String.format(
- DataNodePipeMessages
- .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
- backupId));
- }
- Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize();
- if (!directory.startsWith(backupRoot)) {
- throw new PipeParameterNotValidException(
- String.format(
- DataNodePipeMessages
- .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22,
- backupId));
- }
- final String resume =
- parameters
- .getStringOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE)
- .toLowerCase(Locale.ROOT);
validateSharedDirectoryResumePolicy(parameters);
- final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume);
- if (!append
- && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)
- && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) {
- throw new PipeParameterNotValidException(
- String.format(
- DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9,
- resume));
+ final LogicalBackupSettings settings =
+ new LogicalBackupSettings(parameters, pipeName + "-" + creationTime);
+ includeHeartbeat = settings.includeHeartbeat;
+ unsupportedEventPolicy = settings.unsupported;
+ if (settings.captureOptions.containsKey("start-time")) {
+ backupStartTime =
+ DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ settings.captureOptions.get("start-time"));
}
- if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) {
- directory =
- directory.resolveSibling(
- directory.getFileName()
- + "-new-"
- + System.currentTimeMillis()
- + "-"
- + UUID.randomUUID().toString().substring(0, 8));
+ if (settings.captureOptions.containsKey("end-time")) {
+ backupEndTime =
+ DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
+ settings.captureOptions.get("end-time"));
}
- final long segmentSize =
- parameters.getLongOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE);
- final int maxRecord =
- parameters.getIntOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE);
- final String fsync =
- parameters
- .getStringOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE)
- .toUpperCase(Locale.ROOT);
- final LogicalBackupWriter.FsyncPolicy fsyncPolicy =
- LogicalBackupWriter.FsyncPolicy.valueOf(fsync);
- final int fsyncBatch =
- parameters.getIntOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE);
- final long fsyncPeriod =
- parameters.getLongOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE);
- includeHeartbeat =
- parameters.getBooleanOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE);
- unsupportedEventPolicy =
- parameters
- .getStringOrDefault(
- Arrays.asList(
- PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY,
- PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY),
- PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE)
- .toLowerCase(Locale.ROOT);
-
+ if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) {
+ return;
+ }
+ final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId;
final LogicalBackupManifest manifest = new LogicalBackupManifest();
- manifest.backupId = backupId;
+ manifest.backupId = settings.backupId;
manifest.pipeName = pipeName;
manifest.pipeCreationTime = creationTime;
manifest.sourceClusterId = getSourceClusterId();
@@ -357,16 +259,7 @@ public void customize(
manifest.regionId = regionId;
manifest.streamType = streamType;
manifest.sinkTaskId = streamId;
- writer =
- new LogicalBackupWriter(
- directory,
- manifest,
- segmentSize,
- maxRecord,
- fsyncPolicy,
- fsyncBatch,
- fsyncPeriod,
- append);
+ writer = settings.open(streamType, regionId, manifest);
}
protected String getSourceClusterId() {
@@ -447,9 +340,13 @@ public void transfer(final Event event) throws Exception {
final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event;
transferEnriched(
deletion,
- () ->
- Collections.singletonList(
- PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode())));
+ () -> {
+ final Optional node =
+ boundedDeletion(deletion.getDeleteDataNode(), backupStartTime, backupEndTime);
+ return node.isPresent()
+ ? Collections.singletonList(PipeTransferPlanNodeReq.toTPipeTransferReq(node.get()))
+ : Collections.emptyList();
+ });
} else if (event instanceof PipeSchemaRegionWritePlanEvent) {
final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event;
transferEnriched(
@@ -465,8 +362,8 @@ public void transfer(final Event event) throws Exception {
LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event));
}
} else if (event instanceof PipeTerminateEvent) {
- writer.writeControl(
- LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event));
+ // The framework commits this ordered completion signal. close() writes the final STREAM_END;
+ // a continuous source also emits a historical completion signal before its realtime events.
} else {
handleUnsupported(event);
}
@@ -524,13 +421,14 @@ private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws
}
}
- private List requests(final PipeInsertNodeTabletInsertionEvent event)
+ List requests(final PipeInsertNodeTabletInsertionEvent event)
throws IOException {
final String databaseName =
event.isTableModelEvent()
? event.getTableModelDatabaseName()
: event.getTreeModelDatabaseName();
- if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) {
+ if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)
+ && !event.shouldParseTimeOrPattern()) {
return Collections.singletonList(
PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq(
event.getInsertNode(), databaseName));
@@ -539,6 +437,9 @@ private List requests(final PipeInsertNodeTabletInsertionEvent
final List tablets = event.convertToTablets();
final List requests = new ArrayList<>(tablets.size());
for (int index = 0; index < tablets.size(); index++) {
+ if (tablets.get(index).getRowSize() == 0) {
+ continue;
+ }
requests.add(
PipeTransferTabletRawReqV2.toTPipeTransferReq(
tablets.get(index), event.isAligned(index), databaseName));
@@ -546,14 +447,48 @@ private List requests(final PipeInsertNodeTabletInsertionEvent
return requests;
}
+ static Optional boundedDeletion(
+ final AbstractDeleteDataNode node, final long start, final long end) {
+ if (node instanceof DeleteDataNode) {
+ final DeleteDataNode tree = (DeleteDataNode) node;
+ final long first = Math.max(start, tree.getDeleteStartTime());
+ final long last = Math.min(end, tree.getDeleteEndTime());
+ return first > last
+ ? Optional.empty()
+ : Optional.of(new DeleteDataNode(tree.getPlanNodeId(), tree.getPathList(), first, last));
+ }
+ if (node instanceof RelationalDeleteDataNode) {
+ final RelationalDeleteDataNode table = (RelationalDeleteDataNode) node;
+ final List entries = new ArrayList<>();
+ for (final TableDeletionEntry entry : table.getModEntries()) {
+ final long first = Math.max(start, entry.getStartTime());
+ final long last = Math.min(end, entry.getEndTime());
+ if (first <= last) {
+ entries.add(new TableDeletionEntry(entry.getPredicate(), new TimeRange(first, last)));
+ }
+ }
+ return entries.isEmpty()
+ ? Optional.empty()
+ : Optional.of(
+ new RelationalDeleteDataNode(
+ table.getPlanNodeId(), entries, table.getDatabaseName()));
+ }
+ throw new IllegalArgumentException(
+ String.format(
+ DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97,
+ node.getClass().getName()));
+ }
+
protected void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier)
throws Exception {
if (!event.increaseReferenceCount(getClass().getName())) {
return;
}
try {
- writer.writeEvent(
- eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event));
+ final List requests = requestSupplier.get();
+ if (!requests.isEmpty()) {
+ writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event));
+ }
} finally {
event.decreaseReferenceCount(getClass().getName(), false);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
index 5d677e100472..01d65b54f9c0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
@@ -29,6 +29,7 @@
import org.apache.iotdb.commons.queryengine.common.SqlDialect;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionSourceMetrics;
@@ -136,6 +137,7 @@ public class IoTDBDataRegionSource extends IoTDBSource {
private PipeHistoricalDataRegionSource historicalSource;
private PipeRealtimeDataRegionSource realtimeSource;
+ private boolean logicalBackupCompletionSent;
private DataRegionWatermarkInjector watermarkInjector;
@@ -619,6 +621,15 @@ public Event supply() throws Exception {
}
if (Objects.isNull(event)) {
event = realtimeSource.supply();
+ if (event == null
+ && !logicalBackupCompletionSent
+ && realtimeSource.isLogicalBackupDrained()) {
+ final PipeTerminateEvent completion =
+ new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, regionId, true);
+ completion.increaseReferenceCount(IoTDBDataRegionSource.class.getName());
+ logicalBackupCompletionSent = true;
+ event = completion;
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
index 92ad93e09f1b..9879a3a3ec5d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
@@ -146,6 +146,8 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor {
protected String userName;
protected String cliHostname;
protected boolean skipIfNoPrivileges = true;
+ private long logicalBackupUntil = -1;
+ private final Object logicalBackupFence = new Object();
protected PipeRealtimeDataRegionSource() {
// Do nothing
@@ -218,6 +220,7 @@ public void validate(final PipeParameterValidator validator) throws Exception {
public void customize(
final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration)
throws Exception {
+ logicalBackupUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L);
final PipeRuntimeEnvironment environment = configuration.getRuntimeEnvironment();
final Pair insertionDeletionListeningOptionPair =
@@ -362,6 +365,28 @@ private void clearPendingQueue() {
* @param event the {@link Event} from the {@link StorageEngine}
*/
public final void extract(final PipeRealtimeEvent event) {
+ if (logicalBackupUntil > 0) {
+ synchronized (logicalBackupFence) {
+ if (System.currentTimeMillis() >= logicalBackupUntil) {
+ event.decreaseReferenceCount(PipeRealtimeDataRegionSource.class.getName(), false);
+ return;
+ }
+ extractInternal(event);
+ }
+ } else {
+ extractInternal(event);
+ }
+ }
+
+ public boolean isLogicalBackupDrained() {
+ synchronized (logicalBackupFence) {
+ return logicalBackupUntil > 0
+ && System.currentTimeMillis() >= logicalBackupUntil
+ && pendingQueue.isEmpty();
+ }
+ }
+
+ private void extractInternal(final PipeRealtimeEvent event) {
// The progress report event shall be directly extracted
if (event.getEvent() instanceof ProgressReportEvent) {
extractProgressReportEvent(event);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
index 8c2fd176587d..3711867604e3 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java
@@ -38,6 +38,27 @@
public class PipeSinkSubtaskManagerTest {
+ @Test
+ public void testLogicalBackupUsesOneWriterPerRegion() {
+ for (final BuiltinPipePlugin plugin :
+ new BuiltinPipePlugin[] {
+ BuiltinPipePlugin.LOGICAL_BACKUP_SINK, BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR
+ }) {
+ final Map attributes = new HashMap<>();
+ attributes.put(PipeSinkConstant.SINK_KEY, plugin.getPipePluginName());
+ attributes.put(PipeSinkConstant.SINK_IOTDB_PARALLEL_TASKS_KEY, "3");
+ attributes.put(PipeSinkConstant.SINK_SERIALIZE_BY_REGION_KEY, "false");
+ final PipeParameters parameters = new PipeParameters(attributes);
+ Assert.assertEquals(1, PipeSinkSubtaskManager.calculateSinkSubtaskNum(parameters, -1));
+ Assert.assertNotEquals(
+ PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -1),
+ PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -2));
+ Assert.assertNotEquals(
+ PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE),
+ PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE - 1));
+ }
+ }
+
@Test
public void testSubtasksAreSharedOnlyWithinSamePipe() {
// Initialize the task agent used by PipeEventCommitManager before registering subtasks.
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
index 2354e8309ab9..f2f0407d9b9e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java
@@ -19,19 +19,124 @@
package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration;
+import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
public class LogicalBackupSinkTest {
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testTemporaryValidationDoesNotCreateBackupStreams() throws Exception {
+ final Path directory = temporaryFolder.getRoot().toPath().resolve("backup");
+ final PipeParameters parameters =
+ new PipeParameters(
+ Map.of(
+ "sink.dir",
+ directory.toString(),
+ "sink.backup-id",
+ "backup",
+ "sink.resume",
+ "append"));
+ try (final LogicalBackupSink sink = new LogicalBackupSink("data")) {
+ sink.validate(new PipeParameterValidator(parameters));
+ sink.customize(
+ parameters,
+ new PipeTaskRuntimeConfiguration(new PipeTaskTemporaryRuntimeEnvironment("pipe")));
+ sink.handshake();
+ }
+ Assert.assertFalse(Files.exists(directory));
+ }
+
+ @Test
+ public void testDeleteTimeFilterPreservesPathsAndNeverExpandsTheRange() throws Exception {
+ final DeleteDataNode original =
+ new DeleteDataNode(
+ new PlanNodeId("deletion"),
+ Collections.singletonList(new MeasurementPath("root.backup.d.s1")),
+ 10L,
+ 20L);
+ final DeleteDataNode bounded =
+ (DeleteDataNode) LogicalBackupSink.boundedDeletion(original, 12L, 15L).get();
+ Assert.assertEquals(original.getPathList(), bounded.getPathList());
+ Assert.assertEquals(12L, bounded.getDeleteStartTime());
+ Assert.assertEquals(15L, bounded.getDeleteEndTime());
+ Assert.assertEquals(10L, original.getDeleteStartTime());
+ Assert.assertEquals(20L, original.getDeleteEndTime());
+ Assert.assertFalse(LogicalBackupSink.boundedDeletion(original, 21L, 30L).isPresent());
+ }
+
+ @Test
+ public void testInsertNodeIsFilteredBeforeItIsArchived() throws Exception {
+ final InsertTabletNode node =
+ new InsertTabletNode(
+ new PlanNodeId("backup-test"),
+ new PartialPath("root.backup.d"),
+ true,
+ new String[] {"s1", "s2"},
+ new TSDataType[] {TSDataType.INT32, TSDataType.INT32},
+ new MeasurementSchema[] {
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32)
+ },
+ new long[] {1, 2, 3},
+ null,
+ new Object[] {new int[] {10, 20, 30}, new int[] {40, 50, 60}},
+ 3);
+ final PipeInsertNodeTabletInsertionEvent event =
+ new PipeInsertNodeTabletInsertionEvent(
+ false,
+ "root.backup",
+ node,
+ null,
+ 0,
+ null,
+ new IoTDBTreePattern("root.backup.d.s1"),
+ null,
+ null,
+ null,
+ null,
+ true,
+ 2L,
+ 2L);
+ final List requests = new LogicalBackupSink("data").requests(event);
+ Assert.assertEquals(1, requests.size());
+ final InsertTabletStatement restored =
+ PipeTransferTabletRawReqV2.fromTPipeTransferReq(requests.get(0)).constructStatement();
+ Assert.assertArrayEquals(new String[] {"s1"}, restored.getMeasurements());
+ Assert.assertArrayEquals(new long[] {2}, restored.getTimes());
+ Assert.assertArrayEquals(new int[] {20}, (int[]) restored.getColumns()[0]);
+ Assert.assertTrue(restored.isAligned());
+ }
+
@Test
public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception {
final Map attributes = new HashMap<>();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java
index 4677f2126b6c..b3b7f954ebb0 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java
@@ -34,6 +34,26 @@ public class PipeRealtimeDataRegionSourceTest {
private static final String TEST_REFERENCE_HOLDER =
PipeRealtimeDataRegionSourceTest.class.getName();
+ @Test
+ public void logicalBackupBoundaryRejectsNewEventsAndDrainsAcceptedEvents() throws Exception {
+ try (final ProgressReportTestSource source = new ProgressReportTestSource()) {
+ final java.lang.reflect.Field until =
+ PipeRealtimeDataRegionSource.class.getDeclaredField("logicalBackupUntil");
+ until.setAccessible(true);
+ until.set(source, System.currentTimeMillis() + 60_000L);
+ final PipeRealtimeEvent accepted = createHeartbeatEvent();
+ source.extract(accepted);
+ until.set(source, System.currentTimeMillis() - 1L);
+ final PipeRealtimeEvent late = createHeartbeatEvent();
+ source.extract(late);
+ Assert.assertTrue(late.getEvent().isReleased());
+ Assert.assertFalse(source.isLogicalBackupDrained());
+ Assert.assertSame(accepted, source.supply());
+ accepted.decreaseReferenceCount(TEST_REFERENCE_HOLDER, false);
+ Assert.assertTrue(source.isLogicalBackupDrained());
+ }
+ }
+
@Test
public void progressReportEventReleasesDroppedHeartbeatEvent() throws Exception {
try (final ProgressReportTestSource source = new ProgressReportTestSource()) {
diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
index 1629e2dd0002..deb5f8629fd7 100644
--- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -148,4 +148,14 @@ public final class LogicalBackupMessages {
"Logical backup directory is unavailable: %s";
private LogicalBackupMessages() {}
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 =
+ "Invalid logical backup root manifest, file, checksum or state: %s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB =
+ "Invalid logical backup writer settings.";
+ public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 =
+ "ConfigRegion and DataRegion backups require explicit sink.resume=append and a shared backup directory.";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB =
+ "Invalid logical backup setting: %s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 =
+ "Unsupported logical backup event: %s";
}
diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
index bfc2bed98fed..b4cf5a28670c 100644
--- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java
@@ -148,4 +148,14 @@ public final class LogicalBackupMessages {
"逻辑备份目录不可用：%s";
private LogicalBackupMessages() {}
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 =
+ "逻辑备份根清单、文件、校验和或状态无效：%s";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB =
+ "逻辑备份写入参数无效。";
+ public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 =
+ "ConfigRegion 和 DataRegion 备份必须显式设置 sink.resume=append，并使用共享备份目录。";
+ public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB =
+ "逻辑备份参数无效：%s";
+ public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 =
+ "不支持的逻辑备份事件：%s";
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
index b378a16136c1..f90522acbf82 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java
@@ -60,7 +60,18 @@ public class LogicalBackupArchiveReader {
public List read(final Path source, final boolean allowIncomplete)
throws IOException {
final Path normalizedSource = source.toAbsolutePath().normalize();
- final List manifests = discoverManifests(normalizedSource);
+ LogicalBackupCatalog.rejectSymbolicLinks(normalizedSource);
+ final Path root =
+ Files.isDirectory(normalizedSource) ? normalizedSource : normalizedSource.getParent();
+ final LogicalBackupCatalog catalog =
+ LogicalBackupCatalog.load(
+ Files.isDirectory(normalizedSource)
+ ? normalizedSource.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)
+ : normalizedSource);
+ final List manifests =
+ catalog == null
+ ? discoverManifests(normalizedSource)
+ : catalogManifests(root, catalog, allowIncomplete);
if (manifests.isEmpty()) {
throw new IOException(
String.format(
@@ -72,6 +83,7 @@ public List read(final Path source, final boolean allowIncomplete)
LogicalBackupManifest archiveIdentity = null;
for (final Path manifest : manifests) {
final BackupStream stream = readStream(manifest, allowIncomplete);
+ stream.catalog = catalog;
if (archiveIdentity == null) {
archiveIdentity = stream.getManifest();
} else {
@@ -85,6 +97,9 @@ public List read(final Path source, final boolean allowIncomplete)
}
streams.add(stream);
}
+ if (catalog != null) {
+ validateCatalog(root, catalog, streams);
+ }
streams.sort(
Comparator.comparingInt(
(BackupStream stream) -> streamOrder(stream.getManifest().streamType))
@@ -92,6 +107,104 @@ public List read(final Path source, final boolean allowIncomplete)
return Collections.unmodifiableList(streams);
}
+ private static List catalogManifests(
+ final Path root, final LogicalBackupCatalog catalog, final boolean allowIncomplete)
+ throws IOException {
+ if ((!"SEALED".equals(catalog.status) && !"INCOMPLETE".equals(catalog.status))
+ || (!allowIncomplete
+ && ("INCOMPLETE".equals(catalog.status) || catalog.skippedEventCount > 0))) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ final List manifests = new ArrayList<>();
+ final Set files = new HashSet<>();
+ files.add(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME));
+ for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) {
+ if (entry == null || entry.manifest == null || entry.segments == null) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ final Path manifest = validateCatalogFile(root, entry.manifest, files);
+ if (!LogicalBackupFormat.MANIFEST_FILE_NAME.equals(manifest.getFileName().toString())) {
+ throw LogicalBackupCatalog.invalid(manifest);
+ }
+ manifests.add(manifest);
+ for (final LogicalBackupCatalog.FileEntry segment : entry.segments) {
+ validateCatalogFile(root, segment, files);
+ }
+ }
+ try (final Stream paths = Files.walk(root)) {
+ for (final Path path : paths.toList()) {
+ if (Files.isSymbolicLink(path)
+ || (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)
+ && !files.contains(path)
+ && !".backup.lock".equals(path.getFileName().toString()))) {
+ throw LogicalBackupCatalog.invalid(path);
+ }
+ }
+ }
+ return manifests;
+ }
+
+ private static Path validateCatalogFile(
+ final Path root, final LogicalBackupCatalog.FileEntry entry, final Set files)
+ throws IOException {
+ if (entry == null) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ final Path path = LogicalBackupCatalog.resolve(root, entry.file);
+ if (!files.add(path)
+ || !Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)
+ || entry.sizeBytes != Files.size(path)
+ || !LogicalBackupCatalog.sha256(path).equals(entry.sha256)) {
+ throw LogicalBackupCatalog.invalid(path);
+ }
+ return path;
+ }
+
+ private static void validateCatalog(
+ final Path root, final LogicalBackupCatalog catalog, final List streams)
+ throws IOException {
+ long skipped = 0;
+ final Map counts = new LinkedHashMap<>();
+ final Map highWatermarks = new LinkedHashMap<>();
+ for (int index = 0; index < streams.size(); index++) {
+ final BackupStream stream = streams.get(index);
+ final LogicalBackupManifest manifest = stream.manifest;
+ final LogicalBackupCatalog.StreamEntry entry = catalog.streams.get(index);
+ if (!manifest.streamId.equals(entry.streamId)
+ || !manifest.streamType.equals(entry.streamType)
+ || !manifest.backupId.equals(catalog.backupId)
+ || !java.util.Objects.equals(manifest.pipeName, catalog.pipeName)
+ || !java.util.Objects.equals(manifest.sourceClusterId, catalog.sourceClusterId)
+ || !java.util.Objects.equals(manifest.sourceVersion, catalog.sourceVersion)
+ || !java.util.Objects.equals(manifest.timestampPrecision, catalog.timestampPrecision)
+ || !java.util.Objects.equals(manifest.captureOptions, catalog.captureOptions)
+ || entry.segments.size() != manifest.segments.size()) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ if (("data".equals(manifest.streamType) && !catalog.scope.contains("data"))
+ || ("schema".equals(manifest.streamType) && !catalog.scope.contains("schema"))
+ || ("config".equals(manifest.streamType)
+ && !catalog.scope.contains("config")
+ && !catalog.scope.contains("schema"))) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ for (int segment = 0; segment < manifest.segments.size(); segment++) {
+ if (!LogicalBackupCatalog.resolve(root, entry.segments.get(segment).file)
+ .equals(stream.manifestPath.getParent().resolve(manifest.segments.get(segment).file))) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ }
+ skipped += manifest.skippedEventCount;
+ manifest.operationCounts.forEach((key, value) -> counts.merge(key, value, Long::sum));
+ highWatermarks.put(manifest.streamId, manifest.lastDurableSequence);
+ }
+ if (skipped != catalog.skippedEventCount
+ || !counts.equals(catalog.operationCounts)
+ || !highWatermarks.equals(catalog.sourceHighWatermark)) {
+ throw LogicalBackupCatalog.invalid(root);
+ }
+ }
+
private static int streamOrder(final String streamType) {
if ("config".equalsIgnoreCase(streamType)) {
return 0;
@@ -131,6 +244,10 @@ private static void validateArchiveIdentityField(
private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete)
throws IOException {
final LogicalBackupManifest manifest;
+ LogicalBackupCatalog.rejectSymbolicLinks(manifestPath);
+ if (Files.size(manifestPath) > 16 * 1024 * 1024) {
+ throw LogicalBackupCatalog.invalid(manifestPath);
+ }
try {
manifest =
GSON.fromJson(
@@ -143,7 +260,9 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco
e);
}
validateManifest(manifest, manifestPath);
- if (!allowIncomplete && manifest.skippedEventCount > 0) {
+ if ((!"SEALED".equals(manifest.status) && !"INCOMPLETE".equals(manifest.status))
+ || (!allowIncomplete
+ && (manifest.skippedEventCount > 0 || "INCOMPLETE".equals(manifest.status)))) {
throw new IOException(
String.format(
LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
@@ -158,6 +277,7 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco
final Set listedSegments = new HashSet<>();
long expectedSequence = -1;
long expectedSegmentId = 0;
+ boolean allSealed = true;
for (int index = 0; index < manifest.segments.size(); index++) {
final LogicalBackupManifest.Segment segment = manifest.segments.get(index);
@@ -176,22 +296,23 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco
LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845,
segment.file));
}
- listedSegments.add(segmentPath);
- final boolean lastSegment = index == manifest.segments.size() - 1;
- final LogicalBackupSegmentReader.ScanResult scan =
- segmentReader.scan(segmentPath, allowIncomplete && lastSegment);
+ if (!listedSegments.add(segmentPath)) {
+ throw LogicalBackupCatalog.invalid(segmentPath);
+ }
+ final LogicalBackupSegmentReader.ScanResult scan = segmentReader.scan(segmentPath, false);
if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) {
throw new IOException(
String.format(
LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A,
segmentPath));
}
- if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) {
+ if (!scan.isSealed() || scan.hasOpenEventGroup()) {
throw new IOException(
String.format(
LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF,
segmentPath));
}
+ allSealed &= scan.isSealed();
if (scan.isSealed()) {
validateSegmentManifest(segment, scan, segmentPath);
}
@@ -215,11 +336,11 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco
}
}
- if (!allowIncomplete) {
- validateUnlistedSegments(streamDirectory, listedSegments);
+ validateUnlistedSegments(streamDirectory, listedSegments);
+ if (allSealed) {
final long skippedEventCount =
operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L);
- if (skippedEventCount > 0) {
+ if (!allowIncomplete && skippedEventCount > 0) {
throw new IOException(
String.format(
LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599,
@@ -308,6 +429,7 @@ private static void validateManifest(
|| !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName)
|| !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion)
|| manifest.streamId == null
+ || !List.of("config", "schema", "data").contains(manifest.streamType)
|| manifest.backupId == null
|| manifest.timestampPrecision == null
|| manifest.maxRecordBytes <= 0
@@ -421,6 +543,7 @@ public static class BackupStream {
private final LogicalBackupManifest manifest;
private final List records;
private final List eventGroups;
+ private LogicalBackupCatalog catalog;
private BackupStream(
final Path manifestPath,
@@ -441,6 +564,10 @@ public LogicalBackupManifest getManifest() {
return manifest;
}
+ public LogicalBackupCatalog getCatalog() {
+ return catalog;
+ }
+
public List getRecords() {
return records;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java
new file mode 100644
index 000000000000..3b9d79b1fed1
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** The portable, authoritative root manifest of a backup task. */
+public class LogicalBackupCatalog {
+
+ private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+
+ public String formatName = LogicalBackupFormat.FORMAT_NAME;
+ public String formatVersion = LogicalBackupFormat.FORMAT_VERSION;
+ public String backupId;
+ public String pipeName;
+ public String status = "WRITING";
+ public String sourceClusterId;
+ public String sourceVersion;
+ public String timestampPrecision;
+ public List scope = new ArrayList<>();
+ public List model = new ArrayList<>();
+ public Map filters = new LinkedHashMap<>();
+ public Map captureOptions = new LinkedHashMap<>();
+ public boolean includeAuth;
+ public String mode = "snapshot";
+ public String createdAt = Instant.now().toString();
+ public String capturedAt;
+ public String sealedAt;
+ public Map sourceHighWatermark = new LinkedHashMap<>();
+ public Map operationCounts = new LinkedHashMap<>();
+ public long skippedEventCount;
+ public boolean recovered;
+ public List streams = new ArrayList<>();
+
+ public static class StreamEntry {
+ public String streamId;
+ public String streamType;
+ public FileEntry manifest;
+ public List segments = new ArrayList<>();
+ }
+
+ public static class FileEntry {
+ public String file;
+ public long sizeBytes;
+ public String sha256;
+ }
+
+ public void seal(final Path root, final List backupStreams) throws IOException {
+ collect(root, backupStreams);
+ write(root);
+ }
+
+ public void collect(final Path root, final List backupStreams) throws IOException {
+ streams.clear();
+ operationCounts.clear();
+ sourceHighWatermark.clear();
+ skippedEventCount = 0;
+ recovered = false;
+ for (final BackupStream stream : backupStreams) {
+ final LogicalBackupManifest manifest = stream.getManifest();
+ if (!backupId.equals(manifest.backupId)) {
+ throw invalid(root);
+ }
+ if (streams.isEmpty()) {
+ sourceClusterId = manifest.sourceClusterId;
+ sourceVersion = manifest.sourceVersion;
+ timestampPrecision = manifest.timestampPrecision;
+ }
+ if (!java.util.Objects.equals(pipeName, manifest.pipeName)
+ || !java.util.Objects.equals(sourceClusterId, manifest.sourceClusterId)
+ || !java.util.Objects.equals(sourceVersion, manifest.sourceVersion)
+ || !java.util.Objects.equals(timestampPrecision, manifest.timestampPrecision)
+ || !captureOptions.equals(manifest.captureOptions)) {
+ throw invalid(root);
+ }
+ final StreamEntry entry = new StreamEntry();
+ entry.streamId = manifest.streamId;
+ entry.streamType = manifest.streamType;
+ entry.manifest = describe(root, stream.getManifestPath());
+ for (final LogicalBackupManifest.Segment segment : manifest.segments) {
+ entry.segments.add(
+ describe(root, stream.getManifestPath().getParent().resolve(segment.file)));
+ }
+ streams.add(entry);
+ manifest.operationCounts.forEach(
+ (key, value) -> operationCounts.merge(key, value, Long::sum));
+ sourceHighWatermark.put(manifest.streamId, manifest.lastDurableSequence);
+ skippedEventCount += manifest.skippedEventCount;
+ recovered |= manifest.recovered;
+ }
+ capturedAt = Instant.now().toString();
+ sealedAt = capturedAt;
+ status = skippedEventCount > 0 ? "INCOMPLETE" : "SEALED";
+ }
+
+ public void write(final Path root) throws IOException {
+ rejectSymbolicLinks(root);
+ Files.createDirectories(root);
+ final Path target = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME);
+ if (Files.isSymbolicLink(target)) {
+ throw invalid(target);
+ }
+ final Path temporary = Files.createTempFile(root, ".manifest-", ".tmp");
+ try {
+ final ByteBuffer bytes = StandardCharsets.UTF_8.encode(GSON.toJson(this));
+ try (final FileChannel channel = FileChannel.open(temporary, StandardOpenOption.WRITE)) {
+ while (bytes.hasRemaining()) {
+ channel.write(bytes);
+ }
+ channel.force(true);
+ }
+ try {
+ Files.move(
+ temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (final java.nio.file.AtomicMoveNotSupportedException e) {
+ Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);
+ }
+ } finally {
+ Files.deleteIfExists(temporary);
+ }
+ }
+
+ /** Returns null for a legacy stream manifest; malformed root manifests always fail. */
+ public static LogicalBackupCatalog load(final Path manifest) throws IOException {
+ rejectSymbolicLinks(manifest);
+ if (!Files.isRegularFile(manifest, LinkOption.NOFOLLOW_LINKS)) {
+ return null;
+ }
+ if (Files.size(manifest) > 16 * 1024 * 1024) {
+ throw invalid(manifest);
+ }
+ try {
+ final com.google.gson.JsonObject json =
+ GSON.fromJson(Files.readString(manifest), com.google.gson.JsonObject.class);
+ if (json == null) {
+ throw invalid(manifest);
+ }
+ if (json.has("streamId")) {
+ return null;
+ }
+ final LogicalBackupCatalog catalog = GSON.fromJson(json, LogicalBackupCatalog.class);
+ if (!LogicalBackupFormat.FORMAT_NAME.equals(catalog.formatName)
+ || !LogicalBackupFormat.FORMAT_VERSION.equals(catalog.formatVersion)
+ || catalog.backupId == null
+ || catalog.scope == null
+ || catalog.model == null
+ || catalog.filters == null
+ || catalog.captureOptions == null
+ || catalog.streams == null
+ || catalog.operationCounts == null
+ || catalog.sourceHighWatermark == null
+ || catalog.skippedEventCount < 0
+ || catalog.scope.isEmpty()
+ || !List.of("config", "schema", "data").containsAll(catalog.scope)
+ || catalog.model.isEmpty()
+ || !List.of("tree", "table").containsAll(catalog.model)
+ || !List.of("WRITING", "RECOVERED", "SEALED", "INCOMPLETE").contains(catalog.status)) {
+ throw invalid(manifest);
+ }
+ return catalog;
+ } catch (final JsonParseException | IllegalStateException e) {
+ throw new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3,
+ manifest),
+ e);
+ }
+ }
+
+ public static FileEntry describe(final Path root, final Path path) throws IOException {
+ final Path normalizedRoot = root.toAbsolutePath().normalize();
+ final Path normalizedPath = path.toAbsolutePath().normalize();
+ if (!normalizedPath.startsWith(normalizedRoot)) {
+ throw invalid(path);
+ }
+ final FileEntry entry = new FileEntry();
+ entry.file = normalizedRoot.relativize(normalizedPath).toString().replace('\\', '/');
+ resolve(normalizedRoot, entry.file);
+ entry.sizeBytes = Files.size(normalizedPath);
+ entry.sha256 = sha256(normalizedPath);
+ return entry;
+ }
+
+ public static Path resolve(final Path root, final String relative) throws IOException {
+ if (relative == null
+ || relative.isBlank()
+ || relative.indexOf('\\') >= 0
+ || relative.indexOf(':') >= 0
+ || relative.startsWith("/")
+ || relative.chars().anyMatch(Character::isISOControl)) {
+ throw invalid(root);
+ }
+ for (final String part : relative.split("/", -1)) {
+ if (part.isEmpty() || ".".equals(part) || "..".equals(part)) {
+ throw invalid(root);
+ }
+ }
+ final Path normalizedRoot = root.toAbsolutePath().normalize();
+ final Path path = normalizedRoot.resolve(relative).normalize();
+ if (!path.startsWith(normalizedRoot) || path.equals(normalizedRoot)) {
+ throw invalid(path);
+ }
+ rejectSymbolicLinks(path);
+ return path;
+ }
+
+ public static void rejectSymbolicLinks(final Path path) throws IOException {
+ final Path absolute = path.toAbsolutePath().normalize();
+ Path current = absolute.getRoot();
+ for (final Path part : absolute) {
+ current = current.resolve(part);
+ if (Files.isSymbolicLink(current)) {
+ throw invalid(current);
+ }
+ }
+ }
+
+ public static String sha256(final Path path) throws IOException {
+ final MessageDigest digest = LogicalBackupFormat.newSha256();
+ try (final InputStream input = Files.newInputStream(path)) {
+ final byte[] buffer = new byte[64 * 1024];
+ int length;
+ while ((length = input.read(buffer)) != -1) {
+ digest.update(buffer, 0, length);
+ }
+ }
+ return LogicalBackupFormat.toHex(digest.digest());
+ }
+
+ static IOException invalid(final Path path) {
+ return new IOException(
+ String.format(
+ LogicalBackupMessages
+ .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3,
+ path));
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
index 3742ca88d801..891ed03ff6e8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java
@@ -37,11 +37,15 @@ public class LogicalBackupManifest {
public String timestampPrecision;
public String createdAt;
public String closedAt;
+ public String sealedAt;
+ public Map captureOptions = new LinkedHashMap<>();
public String streamId;
public int regionId;
public String sinkTaskId;
public String streamType;
public String fsyncPolicy;
+ public int fsyncBatchOperations;
+ public long fsyncPeriodMs;
public long segmentSizeBytes;
public int maxRecordBytes;
public long firstSequence = -1;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java
new file mode 100644
index 000000000000..5a95830d03e8
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.commons.i18n.LogicalBackupMessages;
+import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.UUID;
+
+/** Settings shared by ConfigRegion, SchemaRegion and DataRegion backup writers. */
+public final class LogicalBackupSettings {
+
+ public final Path root;
+ public final String backupId;
+ public final String resume;
+ public final long segmentSize;
+ public final int maxRecord;
+ public final LogicalBackupWriter.FsyncPolicy fsync;
+ public final int fsyncBatch;
+ public final long fsyncPeriod;
+ public final boolean includeHeartbeat;
+ public final String unsupported;
+ public final Map captureOptions;
+
+ public LogicalBackupSettings(final PipeParameters parameters, final String defaultBackupId)
+ throws IOException {
+ try {
+ final String directory = value(parameters, "dir", "");
+ if (directory.isBlank()) {
+ throw invalid("dir");
+ }
+ root = Path.of(directory).toAbsolutePath().normalize();
+ LogicalBackupCatalog.rejectSymbolicLinks(root);
+ backupId = value(parameters, "backup-id", defaultBackupId);
+ validateBackupId(backupId);
+ resume = value(parameters, "resume", "fail-if-exists").toLowerCase(Locale.ROOT);
+ if (!List.of("fail-if-exists", "append", "new").contains(resume)) {
+ throw invalid("resume");
+ }
+ segmentSize =
+ Long.parseLong(
+ value(
+ parameters,
+ "segment-size-bytes",
+ Long.toString(PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE)));
+ maxRecord =
+ Integer.parseInt(
+ value(
+ parameters,
+ "max-record-bytes",
+ Integer.toString(
+ PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE)));
+ fsync =
+ LogicalBackupWriter.FsyncPolicy.valueOf(
+ value(parameters, "fsync-policy", "always").toUpperCase(Locale.ROOT));
+ fsyncBatch = Integer.parseInt(value(parameters, "fsync-batch-operations", "1000"));
+ fsyncPeriod = Long.parseLong(value(parameters, "fsync-period-ms", "1000"));
+ includeHeartbeat = booleanValue(parameters, "include-heartbeat");
+ unsupported = value(parameters, "on-unsupported-event", "fail");
+ if (maxRecord <= 0
+ || maxRecord > LogicalBackupFormat.MAX_RECORD_BYTES
+ || segmentSize < maxRecord
+ || fsyncBatch <= 0
+ || fsyncPeriod <= 0
+ || !List.of("fail", "skip").contains(unsupported)
+ || (has(parameters, "fsync-batch-operations")
+ && fsync != LogicalBackupWriter.FsyncPolicy.BATCH)
+ || (has(parameters, "fsync-period-ms")
+ && fsync != LogicalBackupWriter.FsyncPolicy.PERIODIC)) {
+ throw invalid("writer");
+ }
+ captureOptions =
+ new Gson()
+ .fromJson(
+ value(parameters, "capture-options", "{}"),
+ new TypeToken>() {}.getType());
+ if (captureOptions == null) {
+ throw invalid("capture-options");
+ }
+ if (!List.of(
+ "scope",
+ "model",
+ "mode",
+ "include-auth",
+ "include",
+ "exclude",
+ "start-time",
+ "end-time",
+ "until",
+ "segment-size-bytes",
+ "max-record-bytes",
+ "fsync-policy",
+ "on-unsupported",
+ "include-heartbeat",
+ "fsync-batch-operations",
+ "fsync-period-ms")
+ .containsAll(captureOptions.keySet())
+ || captureOptions.containsValue(null)) {
+ throw invalid("capture-options");
+ }
+ } catch (final IllegalArgumentException | com.google.gson.JsonParseException e) {
+ throw new IOException(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB, e);
+ }
+ }
+
+ public void requireSharedResume(final PipeParameters parameters) throws IOException {
+ if (!has(parameters, "resume") || !"append".equals(resume)) {
+ throw new IOException(
+ LogicalBackupMessages
+ .EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90);
+ }
+ }
+
+ public LogicalBackupWriter open(
+ final String streamType, final int regionId, final LogicalBackupManifest manifest)
+ throws IOException {
+ Path directory = root.resolve(backupId).resolve(streamType + "-" + regionId);
+ if ("new".equals(resume)) {
+ directory = directory.resolveSibling(directory.getFileName() + "-new-" + UUID.randomUUID());
+ }
+ manifest.captureOptions = new LinkedHashMap<>(captureOptions);
+ return new LogicalBackupWriter(
+ directory,
+ manifest,
+ segmentSize,
+ maxRecord,
+ fsync,
+ fsyncBatch,
+ fsyncPeriod,
+ "append".equals(resume));
+ }
+
+ public static void validateBackupId(final String value) throws IOException {
+ if (value == null
+ || !value.matches("[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
+ || value.endsWith(".")
+ || value.matches("(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\\..*)?")) {
+ throw invalid("backup-id");
+ }
+ }
+
+ private static boolean booleanValue(final PipeParameters parameters, final String key)
+ throws IOException {
+ final String value = value(parameters, key, "false");
+ if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) {
+ throw invalid(key);
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ private static boolean has(final PipeParameters parameters, final String key) {
+ return parameters.hasAnyAttributes("connector." + key, "sink." + key);
+ }
+
+ private static String value(
+ final PipeParameters parameters, final String key, final String fallback) throws IOException {
+ if (parameters.getAttribute().containsKey("connector." + key)
+ && parameters.getAttribute().containsKey("sink." + key)) {
+ throw invalid(key);
+ }
+ return parameters.getStringOrDefault(
+ Arrays.asList("connector." + key, "sink." + key), fallback);
+ }
+
+ private static IOException invalid(final String key) {
+ return new IOException(
+ String.format(
+ LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB, key));
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
index 9c8ba6f9a9a6..86536587efef 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java
@@ -115,6 +115,7 @@ public LogicalBackupWriter(
<= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE
|| maxRecordBytes <= 0
|| maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES
+ || segmentSizeBytes < maxRecordBytes
|| fsyncBatchOperations <= 0
|| fsyncPeriodMs <= 0) {
throw new IOException(
@@ -144,6 +145,8 @@ public LogicalBackupWriter(
manifest.status = "WRITING";
manifest.createdAt = Instant.now().toString();
manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT);
+ manifest.fsyncBatchOperations = fsyncBatchOperations;
+ manifest.fsyncPeriodMs = fsyncPeriodMs;
manifest.segmentSizeBytes = segmentSizeBytes;
manifest.maxRecordBytes = maxRecordBytes;
writeManifest();
@@ -759,6 +762,17 @@ private void loadExistingManifest() throws IOException {
"fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy);
validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes);
validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes);
+ if (existing.fsyncBatchOperations > 0) {
+ validateManifestConfiguration(
+ "fsyncBatchOperations", fsyncBatchOperations, existing.fsyncBatchOperations);
+ }
+ if (existing.fsyncPeriodMs > 0) {
+ validateManifestConfiguration("fsyncPeriodMs", fsyncPeriodMs, existing.fsyncPeriodMs);
+ }
+ validateManifestIdentity(
+ "captureOptions",
+ GSON.toJson(manifest.captureOptions),
+ GSON.toJson(existing.captureOptions));
if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) {
throw new IOException(
String.format(
@@ -799,7 +813,11 @@ private void loadExistingManifest() throws IOException {
: existing.operationCounts;
manifest.createdAt = existing.createdAt;
manifest.closedAt = existing.closedAt;
+ manifest.sealedAt = existing.sealedAt;
+ manifest.captureOptions = existing.captureOptions;
manifest.fsyncPolicy = existing.fsyncPolicy;
+ manifest.fsyncBatchOperations = fsyncBatchOperations;
+ manifest.fsyncPeriodMs = fsyncPeriodMs;
manifest.segmentSizeBytes = existing.segmentSizeBytes;
manifest.maxRecordBytes = existing.maxRecordBytes;
manifest.lastEventGroupId = existing.lastEventGroupId;
@@ -904,8 +922,9 @@ public synchronized void close() throws IOException {
}
sealSegment();
}
- manifest.status = manifest.recovered ? "RECOVERED" : "SEALED";
+ manifest.status = manifest.skippedEventCount > 0 ? "INCOMPLETE" : "SEALED";
manifest.closedAt = Instant.now().toString();
+ manifest.sealedAt = manifest.closedAt;
manifest.lastDurableSequence = manifest.lastSequence;
writeManifest();
} finally {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
index d4d55ba7dac7..7ef4cf1b5217 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
@@ -56,6 +56,8 @@
@TableModel
public abstract class IoTDBNonDataRegionSource extends IoTDBSource {
+ public static final long LOGICAL_BACKUP_COMPLETED_INDEX = Long.MAX_VALUE - 1;
+
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBNonDataRegionSource.class);
protected IoTDBTreePatternOperations treePattern;
@@ -66,6 +68,14 @@ public abstract class IoTDBNonDataRegionSource extends IoTDBSource {
// filled. Used only for metric framework.
private int historicalEventsCount = 0;
+ // Logical backup snapshots must acknowledge all metadata before the coordinator drops the pipe.
+ // Leave room for queue maintenance to calculate index + 1 without overflowing.
+ private boolean finiteSnapshot;
+ private boolean snapshotOnly;
+ private long captureUntil;
+ private volatile long captureEndIndex = Long.MAX_VALUE;
+ private boolean snapshotCompletionReported;
+
private ConcurrentIterableLinkedQueue.DynamicIterator iterator;
// If close() is called, hasBeenClosed will be set to true even if the extractor is started again.
@@ -96,6 +106,9 @@ public void customize(
}
treePattern = (IoTDBTreePatternOperations) pattern;
tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters);
+ snapshotOnly = parameters.getBooleanOrDefault("source.logical-backup.snapshot", false);
+ captureUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L);
+ finiteSnapshot = snapshotOnly || captureUntil > 0;
}
@Override
@@ -107,6 +120,13 @@ public void start() throws Exception {
final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex();
warnIfUnexpectedHybridProgressIndex(progressIndex);
final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex);
+ if (finiteSnapshot
+ && metaProgressIndex != null
+ && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) {
+ snapshotCompletionReported = true;
+ super.start();
+ return;
+ }
final long nextIndex =
Objects.isNull(metaProgressIndex)
// If the index is invalid, the queue is seen as cleared before and thus
@@ -115,6 +135,12 @@ public void start() throws Exception {
? getNextIndexAfterSnapshot()
: metaProgressIndex.getIndex() + 1;
iterator = getListeningQueue().newIterator(nextIndex);
+ if (captureUntil > 0) {
+ java.util.concurrent.CompletableFuture.delayedExecutor(
+ Math.max(0L, captureUntil - System.currentTimeMillis()),
+ java.util.concurrent.TimeUnit.MILLISECONDS)
+ .execute(() -> captureEndIndex = getListeningQueue().getTailIndex());
+ }
super.start();
}
@@ -171,6 +197,9 @@ public EnrichedEvent supply() throws Exception {
return null;
}
}
+ if (snapshotCompletionReported) {
+ return null;
+ }
// Check whether snapshot being parsed exists
PipeWritePlanEvent realtimeEvent = lastEvent;
@@ -197,7 +226,9 @@ public EnrichedEvent supply() throws Exception {
Long.MIN_VALUE,
Long.MAX_VALUE);
- if (canSkipSnapshotPrivilegeCheck(historicalEvent)) {
+ // Portable backups contain logical metadata plans, even when the user can read the whole
+ // snapshot. Snapshot file transfer remains available to ordinary replication pipes.
+ if (!finiteSnapshot && canSkipSnapshotPrivilegeCheck(historicalEvent)) {
if (historicalEvents.isEmpty()) {
// We only report progress for the last snapshot event.
historicalEvent.bindProgressIndex(new MetaProgressIndex(iterator.getNextIndex() - 1));
@@ -221,7 +252,20 @@ public EnrichedEvent supply() throws Exception {
// Realtime
if (Objects.isNull(realtimeEvent)) {
+ if (finiteSnapshot
+ && (snapshotOnly || captureUntil <= 0 || iterator.getNextIndex() >= captureEndIndex)) {
+ final ProgressReportEvent completion =
+ new ProgressReportEvent(pipeName, creationTime, pipeTaskMeta);
+ completion.bindProgressIndex(new MetaProgressIndex(LOGICAL_BACKUP_COMPLETED_INDEX));
+ completion.increaseReferenceCount(IoTDBNonDataRegionSource.class.getName());
+ snapshotCompletionReported = true;
+ return completion;
+ }
realtimeEvent = (PipeWritePlanEvent) iterator.next(getMaxBlockingTimeMs());
+ if (captureUntil > 0 && iterator.getNextIndex() > captureEndIndex) {
+ // A blocking read may wake with an event appended after the boundary was captured.
+ realtimeEvent = null;
+ }
}
if (Objects.isNull(realtimeEvent)) {
return null;
@@ -305,6 +349,11 @@ public long getUnTransferredEventCount() {
final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex();
warnIfUnexpectedHybridProgressIndex(progressIndex);
final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex);
+ if (finiteSnapshot
+ && metaProgressIndex != null
+ && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) {
+ return 0L;
+ }
return Objects.nonNull(metaProgressIndex)
? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1
: getListeningQueue().getSize() + historicalEventsCount;
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
index 10db588f2c3c..3702850597d9 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java
@@ -91,7 +91,7 @@ public void testUnlistedSegmentIsRejected() throws Exception {
}
@Test
- public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception {
+ public void testIncompleteOptInCannotBypassMissingFooter() throws Exception {
final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
writeStream(directory, "stream", "data", 4096);
final Path segment = onlySegment(directory);
@@ -100,9 +100,7 @@ public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception {
}
assertReadFails(directory, false);
- final List streams = new LogicalBackupArchiveReader().read(directory, true);
- Assert.assertEquals(1, streams.size());
- Assert.assertEquals(1, streams.get(0).getEventGroups().size());
+ assertReadFails(directory, true);
}
@Test
@@ -216,6 +214,103 @@ public void testDuplicateEventGroupIdIsRejected() throws Exception {
assertReadFails(directory, false);
}
+ @Test
+ public void testRootManifestIsPortableAndOrdersAllStreams() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data-7"), "data-7", "data", 4096);
+ writeStream(root.resolve("schema-3"), "schema-3", "schema", 4096);
+ writeStream(root.resolve("config-0"), "config-0", "config", 4096);
+ final LogicalBackupCatalog catalog = sealCatalog(root);
+
+ final List streams =
+ new LogicalBackupArchiveReader().read(root.resolve("manifest.json"), false);
+ Assert.assertEquals("config", streams.get(0).getManifest().streamType);
+ Assert.assertEquals("schema", streams.get(1).getManifest().streamType);
+ Assert.assertEquals("data", streams.get(2).getManifest().streamType);
+ Assert.assertEquals(3, catalog.streams.size());
+ Assert.assertEquals("SEALED", streams.get(0).getCatalog().status);
+ for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) {
+ Assert.assertFalse(Path.of(entry.manifest.file).isAbsolute());
+ Assert.assertEquals(64, entry.manifest.sha256.length());
+ Assert.assertEquals(64, entry.segments.get(0).sha256.length());
+ }
+ }
+
+ @Test
+ public void testRootManifestRejectsUnlistedFile() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data", "data", 4096);
+ sealCatalog(root);
+ Files.writeString(root.resolve("unlisted.txt"), "not part of the backup");
+ assertReadFails(root, false);
+ assertReadFails(root, true);
+ }
+
+ @Test
+ public void testIncompleteOptInCannotBypassRootChecksum() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data", "data", 4096);
+ sealCatalog(root);
+ final Path segment = onlySegment(root.resolve("data"));
+ try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) {
+ file.seek(LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.RECORD_HEADER_SIZE);
+ file.writeByte(0);
+ }
+ assertReadFails(root, false);
+ assertReadFails(root, true);
+ }
+
+ @Test
+ public void testRootManifestRejectsTraversalAndDuplicateFiles() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ writeStream(root.resolve("data"), "data", "data", 4096);
+ final LogicalBackupCatalog catalog = sealCatalog(root);
+ final String file = catalog.streams.get(0).manifest.file;
+ catalog.streams.get(0).manifest.file = "../outside/manifest.json";
+ catalog.write(root);
+ assertReadFails(root, true);
+ catalog.streams.get(0).manifest.file = file;
+ catalog.streams.add(catalog.streams.get(0));
+ catalog.write(root);
+ assertReadFails(root, true);
+ }
+
+ @Test
+ public void testIncompleteStatusRequiresExplicitOptIn() throws Exception {
+ final Path root = temporaryFolder.newFolder().toPath();
+ final Path directory = root.resolve("data");
+ try (final LogicalBackupWriter writer = writer(directory, manifest("data", "data"), 4096)) {
+ writer.recordSkippedEvent(1, "unsupported");
+ }
+ final LogicalBackupCatalog catalog = sealCatalog(root);
+ Assert.assertEquals("INCOMPLETE", catalog.status);
+ Assert.assertEquals(1, catalog.skippedEventCount);
+ assertReadFails(root, false);
+ Assert.assertEquals(1, new LogicalBackupArchiveReader().read(root, true).size());
+ }
+
+ @Test
+ public void testAllowIncompleteStillChecksSealedCounters() throws Exception {
+ final Path directory = temporaryFolder.newFolder().toPath().resolve("backup");
+ writeStream(directory, "data", "data", 4096);
+ final Path path = directory.resolve("manifest.json");
+ final LogicalBackupManifest manifest =
+ GSON.fromJson(Files.readString(path), LogicalBackupManifest.class);
+ manifest.operationCounts.clear();
+ Files.writeString(path, GSON.toJson(manifest));
+ assertReadFails(directory, true);
+ }
+
+ private static LogicalBackupCatalog sealCatalog(final Path root) throws IOException {
+ final LogicalBackupCatalog catalog = new LogicalBackupCatalog();
+ catalog.backupId = "backup";
+ catalog.pipeName = "pipe";
+ catalog.scope = Arrays.asList("config", "schema", "data");
+ catalog.model = Arrays.asList("tree", "table");
+ catalog.seal(root, new LogicalBackupArchiveReader().read(root, true));
+ return catalog;
+ }
+
private static void writeStream(
final Path directory, final String streamId, final String streamType, final long segmentSize)
throws Exception {
@@ -247,7 +342,7 @@ private static LogicalBackupWriter writer(
directory,
manifest,
segmentSize,
- MAX_RECORD_BYTES,
+ (int) Math.min(MAX_RECORD_BYTES, segmentSize),
LogicalBackupWriter.FsyncPolicy.ALWAYS,
1,
1,
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java
new file mode 100644
index 000000000000..cdef5ccafd8c
--- /dev/null
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.logicalbackup;
+
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class LogicalBackupSettingsTest {
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testInvalidSettingsFailBeforeCreatingDirectory() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath().resolve("backup");
+ for (final Map invalid :
+ Arrays.asList(
+ Map.of("sink.backup-id", "../outside"),
+ Map.of("sink.backup-id", "bad\\path"),
+ Map.of("sink.backup-id", "CON"),
+ Map.of("sink.segment-size-bytes", "1024"),
+ Map.of("sink.max-record-bytes", "268435457"),
+ Map.of("sink.fsync-policy", "batch", "sink.fsync-batch-operations", "0"),
+ Map.of("sink.fsync-period-ms", "1000"),
+ Map.of("sink.include-heartbeat", "maybe"),
+ Map.of("sink.on-unsupported-event", "ignore"))) {
+ final Map values = new HashMap<>(invalid);
+ values.put("sink.dir", root.toString());
+ try {
+ new LogicalBackupSettings(new PipeParameters(values), "backup");
+ Assert.fail(invalid.toString());
+ } catch (final IOException expected) {
+ Assert.assertFalse(Files.exists(root));
+ }
+ }
+ }
+
+ @Test
+ public void testAllStreamsUseTheSameWriterSettingsAndAppendIdentity() throws Exception {
+ final Path root = temporaryFolder.getRoot().toPath().resolve("backup");
+ final Map values = new HashMap<>();
+ values.put("sink.dir", root.toString());
+ values.put("sink.resume", "append");
+ values.put("sink.backup-id", "backup-id");
+ values.put("sink.segment-size-bytes", "8192");
+ values.put("sink.max-record-bytes", "1024");
+ values.put("sink.fsync-policy", "periodic");
+ values.put("sink.fsync-period-ms", "12");
+ values.put("sink.capture-options", "{\"scope\":\"config,schema,data\"}");
+ final LogicalBackupSettings settings =
+ new LogicalBackupSettings(new PipeParameters(values), "backup");
+ for (final String type : Arrays.asList("config", "schema", "data")) {
+ final LogicalBackupManifest manifest = new LogicalBackupManifest();
+ manifest.backupId = "backup-id";
+ manifest.streamId = type;
+ manifest.streamType = type;
+ manifest.timestampPrecision = "ms";
+ try (final LogicalBackupWriter writer = settings.open(type, 1, manifest)) {
+ Assert.assertEquals(8192, writer.getManifest().segmentSizeBytes);
+ Assert.assertEquals(1024, writer.getManifest().maxRecordBytes);
+ Assert.assertEquals("periodic", writer.getManifest().fsyncPolicy);
+ }
+ values.put("sink.capture-options", "{\"scope\":\"data\"}");
+ try {
+ new LogicalBackupSettings(new PipeParameters(values), "backup").open(type, 1, manifest);
+ Assert.fail();
+ } catch (final IOException expected) {
+ // A different scope cannot silently append to an existing stream.
+ }
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
index ec7d5d63ed9f..b4b97c233fb8 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java
@@ -326,14 +326,7 @@ public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Excep
final LogicalBackupManifest manifest = manifest();
try (final LogicalBackupWriter writer =
new LogicalBackupWriter(
- directory,
- manifest,
- 500,
- MAX_RECORD_BYTES,
- LogicalBackupWriter.FsyncPolicy.NONE,
- 1,
- 1,
- false)) {
+ directory, manifest, 500, 500, LogicalBackupWriter.FsyncPolicy.NONE, 1, 1, false)) {
writer.writeEvent(
UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m");
writer.writeEvent(
@@ -533,7 +526,7 @@ private static LogicalBackupWriter writer(
directory,
manifest(),
segmentSize,
- MAX_RECORD_BYTES,
+ (int) Math.min(MAX_RECORD_BYTES, segmentSize),
LogicalBackupWriter.FsyncPolicy.ALWAYS,
1,
1,
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java
index 3bfbd6dcefdd..bcfce467d3ff 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java
@@ -28,8 +28,10 @@
import org.apache.iotdb.commons.exception.auth.AccessDeniedException;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.event.PipeSnapshotEvent;
import org.apache.iotdb.commons.pipe.event.PipeWritePlanEvent;
+import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.junit.Assert;
import org.junit.Test;
@@ -39,10 +41,62 @@
import java.io.IOException;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.Optional;
public class IoTDBNonDataRegionSourceTest {
+ @Test
+ public void testLogicalBackupParsesMetadataInsteadOfTransferringSnapshotFiles() throws Exception {
+ final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class);
+ final TestNonDataRegionSource source =
+ new TestNonDataRegionSource(queue, new PipeTaskMeta(new MetaProgressIndex(10L), 0));
+ final java.lang.reflect.Field finite =
+ IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot");
+ finite.setAccessible(true);
+ finite.set(source, true);
+ final java.lang.reflect.Field snapshots =
+ IoTDBNonDataRegionSource.class.getDeclaredField("historicalEvents");
+ snapshots.setAccessible(true);
+ snapshots.set(
+ source,
+ new LinkedList<>(
+ Collections.singletonList(
+ Mockito.mock(PipeSnapshotEvent.class, Mockito.RETURNS_SELF))));
+ source.skipSnapshotPrivilegeCheck = true;
+ source.hasBeenStarted.set(true);
+ Assert.assertTrue(source.supply() instanceof ProgressReportEvent);
+ Assert.assertTrue(source.snapshotParsed);
+ Mockito.verifyZeroInteractions(queue);
+ }
+
+ @Test
+ public void testFiniteSnapshotReportsOrderedCompletionAndNeverReadsRealtime() throws Exception {
+ final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class);
+ final PipeTaskMeta task = new PipeTaskMeta(new MetaProgressIndex(10L), 0);
+ final TestNonDataRegionSource source = new TestNonDataRegionSource(queue, task);
+ final java.lang.reflect.Field finite =
+ IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot");
+ finite.setAccessible(true);
+ finite.set(source, true);
+ source.hasBeenStarted.set(true);
+ final EnrichedEvent completion = source.supply();
+ Assert.assertTrue(completion instanceof ProgressReportEvent);
+ Assert.assertEquals(
+ new MetaProgressIndex(IoTDBNonDataRegionSource.LOGICAL_BACKUP_COMPLETED_INDEX),
+ completion.getProgressIndex());
+ Assert.assertNull(source.supply());
+ task.updateProgressIndex(completion.getProgressIndex());
+ Assert.assertEquals(0L, source.getUnTransferredEventCount());
+ Mockito.verifyZeroInteractions(queue);
+
+ final TestNonDataRegionSource restarted = new TestNonDataRegionSource(queue, task);
+ finite.set(restarted, true);
+ restarted.start();
+ Assert.assertNull(restarted.supply());
+ Mockito.verifyZeroInteractions(queue);
+ }
+
@Test
public void testStartWithStateWrappedHybridProgressIndex() throws Exception {
final AbstractPipeListeningQueue listeningQueue =
@@ -94,6 +148,8 @@ public void testGetUnTransferredEventCountWithHybridProgressIndexWithoutMetaInde
private static final class TestNonDataRegionSource extends IoTDBNonDataRegionSource {
private final AbstractPipeListeningQueue listeningQueue;
+ private boolean skipSnapshotPrivilegeCheck;
+ private boolean snapshotParsed;
private TestNonDataRegionSource(
final AbstractPipeListeningQueue listeningQueue, final PipeTaskMeta pipeTaskMeta) {
@@ -123,13 +179,13 @@ protected long getMaxBlockingTimeMs() {
@Override
protected boolean canSkipSnapshotPrivilegeCheck(final PipeSnapshotEvent event) {
- return false;
+ return skipSnapshotPrivilegeCheck;
}
@Override
protected void initSnapshotGenerator(final PipeSnapshotEvent event)
throws IOException, IllegalPathException {
- // Do nothing
+ snapshotParsed = true;
}
@Override