diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java new file mode 100644 index 0000000000000..4629bcb887506 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.tool.pipe.PipeLogicalBackupTool; + +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeLogicalBackupIT extends AbstractPipeDualTreeModelManualIT { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Override + protected void setupConfig() { + super.setupConfig(); + for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) { + env.getConfig().getConfigNodeJVMConfig().setInitHeapSize(128).setMaxHeapSize(256); + env.getConfig().getDataNodeJVMConfig().setInitHeapSize(256).setMaxHeapSize(768); + } + } + + @Test + public void testBoundedContinuousTableCapture() throws Exception { + try (final Connection connection = senderEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database backup_db"); + statement.execute("use backup_db"); + statement.execute("create table measurements(device STRING TAG, reading INT32 FIELD)"); + statement.execute("create table other_table(device STRING TAG, reading INT32 FIELD)"); + statement.execute("insert into measurements(time,device,reading) values(1,'d1',10)"); + statement.execute("insert into other_table(time,device,reading) values(1,'d1',100)"); + statement.execute("flush"); + } + final String pipeName = "logical_backup_continuous_table"; + final CompletableFuture writes = + CompletableFuture.runAsync( + () -> { + try { + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .until( + () -> { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = + statement.executeQuery("show pipe " + pipeName)) { + return result.next(); + } + }); + try (final Connection connection = senderEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("use backup_db"); + statement.execute( + "insert into measurements(time,device,reading) values(2,'d1',20)"); + statement.execute( + "insert into measurements(time,device,reading) values(4,'d1',40)"); + statement.execute( + "insert into other_table(time,device,reading) values(2,'d1',200)"); + statement.execute("delete from measurements where time <= 1"); + } + } catch (final Exception e) { + throw new CompletionException(e); + } + }); + final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath(); + runConnected( + senderEnv, + "dump", + "--backup-dir", + parent.toString(), + "--backup-id", + "bounded", + "--pipe-name", + pipeName, + "--scope", + "schema,data", + "--model", + "table", + "--include", + "backup_db/measurements", + "--start-time", + "0", + "--end-time", + "3", + "--mode", + "continuous", + "--until", + Instant.now().plusSeconds(20).toString(), + "--timeout-seconds", + "180", + "--poll-interval-seconds", + "1"); + writes.get(30, TimeUnit.SECONDS); + final Path backup = parent.resolve("bounded"); + final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json")); + Assert.assertEquals("SEALED", catalog.status); + Assert.assertEquals("continuous", catalog.mode); + assertLogicalRequests(backup); + runConnected(receiverEnv, "restore", "--input", backup.toString()); + try (final Connection connection = receiverEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("use backup_db"); + try (final ResultSet result = statement.executeQuery("select * from measurements")) { + Assert.assertTrue(result.next()); + Assert.assertEquals(2L, result.getLong("time")); + Assert.assertEquals("d1", result.getString("device")); + Assert.assertEquals(20, result.getInt("reading")); + Assert.assertFalse(result.next()); + } + try (final ResultSet result = statement.executeQuery("show tables")) { + while (result.next()) { + Assert.assertNotEquals("other_table", result.getString(1)); + } + } + } + } + + @Test + public void testDumpPackPreflightRestoreWithScopeAndTimeFilters() throws Exception { + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "create database root.backup", + "create aligned timeseries root.backup.d(s1 INT32, s2 BOOLEAN)", + "insert into root.backup.d(time,s1,s2) aligned values(1,10,true)", + "insert into root.backup.d(time,s2) aligned values(2,true)", + "insert into root.backup.d(time,s1,s2) aligned values(3,30,false)", + "insert into root.backup.d(time,s1,s2) aligned values(4,40,true)", + "create database root.unselected", + "create timeseries root.unselected.d.s with datatype=INT32,encoding=PLAIN", + "insert into root.unselected.d(time,s) values(2,100)", + "create user logical_backup_excluded_user 'test_password'", + "flush"), + null); + final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath(); + runConnected( + senderEnv, + "dump", + "--backup-dir", + parent.toString(), + "--backup-id", + "snapshot", + "--scope", + "config,schema,data", + "--model", + "tree", + "--include", + "root.backup.**", + "--start-time", + "2", + "--end-time", + "3", + "--timeout-seconds", + "180", + "--poll-interval-seconds", + "1"); + final Path backup = parent.resolve("snapshot"); + final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json")); + Assert.assertEquals("SEALED", catalog.status); + Assert.assertFalse(catalog.includeAuth); + Assert.assertFalse(catalog.streams.isEmpty()); + assertLogicalRequests(backup); + final Path zip = parent.resolve("snapshot.zip"); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run(new String[] {"verify", "--input", backup.toString(), "--json"})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", backup.toString(), "--output", zip.toString()})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"restore", "--input", zip.toString(), "--dry-run"})); + runConnected(receiverEnv, "restore", "--input", zip.toString(), "--dry-run"); + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = statement.executeQuery("show timeseries root.backup.**")) { + Assert.assertFalse(result.next()); + } + final Path checkpoint = parent.resolve("restore-checkpoint.json"); + runConnected( + receiverEnv, "restore", "--input", zip.toString(), "--checkpoint", checkpoint.toString()); + runConnected( + receiverEnv, + "restore", + "--input", + zip.toString(), + "--checkpoint", + checkpoint.toString(), + "--resume"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.backup.**", + "Time,root.backup.d.s1,root.backup.d.s2,", + new HashSet<>(Arrays.asList("2,null,true,", "3,30,false,"))); + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try (final ResultSet result = statement.executeQuery("show timeseries root.unselected.**")) { + Assert.assertFalse(result.next()); + } + try (final ResultSet result = statement.executeQuery("list user")) { + while (result.next()) { + for (int column = 1; column <= result.getMetaData().getColumnCount(); column++) { + Assert.assertNotEquals("logical_backup_excluded_user", result.getString(column)); + } + } + } + } + } + + private static void assertLogicalRequests(final Path backup) throws Exception { + final Set logicalRequests = + Set.of( + PipeRequestType.TRANSFER_CONFIG_PLAN.getType(), + PipeRequestType.TRANSFER_PLAN_NODE.getType(), + PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), + PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType()); + new LogicalBackupArchiveReader() + .read(backup, false).stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .flatMap(group -> group.getRequests().stream()) + .forEach( + record -> Assert.assertTrue(logicalRequests.contains(record.getRequestType()))); + } + + private static void runConnected(final BaseEnv env, final String command, final String... options) + throws Exception { + final List args = + new ArrayList<>( + Arrays.asList( + command, + "--host", + env.getIP(), + "--port", + env.getPort(), + "--user", + "root", + "--password-stdin")); + args.addAll(Arrays.asList(options)); + final InputStream previous = System.in; + try { + System.setIn(new ByteArrayInputStream("root\n".getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(0, PipeLogicalBackupTool.run(args.toArray(new String[0]))); + } finally { + System.setIn(previous); + } + } +} diff --git a/iotdb-client/cli/LOGICAL_BACKUP.md b/iotdb-client/cli/LOGICAL_BACKUP.md new file mode 100644 index 0000000000000..2fa605314e425 --- /dev/null +++ b/iotdb-client/cli/LOGICAL_BACKUP.md @@ -0,0 +1,138 @@ + + +# Logical backup + +Use `tools/logical-backup.sh` on Linux/macOS or +`tools/windows/logical-backup.bat` on Windows. `--help` lists the supported options. +Commands exit with 0 on success and 1 on failure. Errors go to stderr; `--json` +provides a machine-readable result on stdout. + +## Capture, check, transport, restore + +```sh +tools/logical-backup.sh dump --host source-host --port 6667 --user root \ + --password-stdin --backup-dir /mnt/shared-backups --backup-id migration_001 \ + --scope config,schema,data --model tree,table --mode snapshot + +tools/logical-backup.sh inspect --input /mnt/shared-backups/migration_001 --json +tools/logical-backup.sh verify --input /mnt/shared-backups/migration_001 --json +tools/logical-backup.sh pack --input /mnt/shared-backups/migration_001 \ + --output /archives/migration_001.zip + +# Local verification; no target connection is opened. +tools/logical-backup.sh restore --input /archives/migration_001.zip --dry-run + +# Target version, precision, authentication and protocol capability checks. +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin --dry-run + +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin \ + --checkpoint /archives/migration_001.restore.json + +# After an interrupted restore, retain all connection and input options and add --resume. +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin \ + --checkpoint /archives/migration_001.restore.json --resume +``` + +Passwords are read from stdin or `--password-env NAME`. Plaintext password +arguments are rejected. Manifests and checkpoints contain no connection secrets. + +`dump` generates a directory, and `pack` only packages an existing directory. +`pack` rejects connection options. Output and checkpoint files must be outside +the input backup, and existing output archives are never overwritten. + +All source ConfigNodes, DataNodes, and the CLI must access the same backup +directory at the specified path. This is a shared filesystem workflow, not a +remote download protocol. Do not modify a directory while it is being checked, +packed or restored. + +## Scope and boundaries + +`--scope` is required and selects `config`, `schema`, `data`, or a comma-separated +combination. Schema capture includes its ConfigRegion definitions, such as +databases and tables. `--model` defaults to `tree,table`. + +Authorization objects are excluded by default. Capturing them requires +`--scope config` (possibly combined with other scopes) and `--include-auth true`. +Restoring them also requires `--include-auth true`; inspect/dry-run reports this +selection. Legacy config archives are conservatively treated as containing +authorization objects. + +Object filters require a single model. Tree data accepts source path inclusion +and exclusion lists; when capturing config/schema, the source supports a single +prefix or full path without exclusions. Table filters use +`database-regex/table-regex`; exclusions must match either whole databases or +whole table-name patterns. Unsupported combinations fail before task creation. +Data `--start-time` and `--end-time` are inclusive and use source timestamp +precision. These filters do not remove the schema needed to describe selected +objects. + +`snapshot` captures finite historical streams. `continuous` requires an ISO-8601 +`--until` boundary. Each source stops accepting realtime work at its capture +boundary, drains accepted work, and acknowledges completion before task removal. +The boundary uses source clocks; streams do not represent a global transaction. +`--stop-sequence` is reserved and rejected because this server does not expose a +stable global source sequence. `sourceHighWatermark` records each stream's durable +archive sequence; it is not a cluster-wide source log position. + +The root `manifest.json` is the portable entry point. It declares scope, +models, filters, capture settings, streams, relative file paths, full-file SHA-256 +digests, counts, status and capture/seal times. Stream manifests describe the +versioned `.pwal` frames, CRCs, segment digests and committed event groups. Legacy +stream-only directories remain readable; packing them adds a root manifest +without changing the input. + +Config and Schema snapshots are decoded into logical plans before capture; +data files are decoded into logical tablet requests. Each Region has its own +ordered stream and writer. Pipe parameter validation creates no backup files. + +## Interrupted work and conflicts + +`dump --resume fail-if-exists` is the default. `new` chooses a new backup identity +if the requested directory exists. `append` requires the same identity, scope +and writing settings. A timed-out command reports the backup, directory and +Pipe identity; query `SHOW PIPE` or rerun the same dump with `--resume append`. +An already sealed matching backup is returned after verification. + +Writer sizes, fsync policy and batch/period combinations are validated before +writing. Unknown events fail by default; `--on-unsupported skip` records a gap +and seals the result as `INCOMPLETE`. Inspect, verify, pack and restore reject +incomplete backups unless `--allow-incomplete` is explicit. That flag never +bypasses checksum errors, truncated segments, missing files or invalid groups. + +Restore validates locally before connecting, then replays config, schema and +data, sorting equal-type streams by stream ID. Dry-run sends no replay requests. +Target preflight checks version, precision, authentication and protocol +handshakes; object conflicts and object-level permissions are checked during +replay. Only `--conflict fail` is supported. Target errors stop replay and retain +the checkpoint; no schema is silently deleted, overwritten or skipped. + +Checkpoints bind the archive/content digest, backup format and identity, target +host/port/user, completed groups and the next request within a group. A request +that may have succeeded immediately before an interrupted checkpoint update can +be retried; only that uncertain request accepts an idempotent-conflict response. +There is no transaction across requests or streams. Completed checkpoints remain +available for audit and repeat invocation. + +`export` and `import` are deprecated compatibility aliases. Offline `export` is +`pack`; connected `export` performs `dump` then `pack`; `import` is `restore`. +Both aliases print a migration notice on stderr. diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index 5de247a084a09..c9824ef4aec11 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -124,6 +124,10 @@ commons-cli commons-cli + + com.google.code.gson + gson + org.jline jline @@ -204,6 +208,7 @@ org.apache.iotdb:iotdb-server org/apache/iotdb/db/utils/* + org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.class diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 0b0f30d970d95..100381d53ea66 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -100,4 +100,117 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "Input time format {} is not supported, "; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n"; + // Pipe logical backup tool + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "Logical backup command failed: %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "Unknown logical backup command: %s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "--output is required for logical backup export"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "Stream %s: %d records, %d committed event groups"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "Logical backup verified: %d streams, %d records, %d committed event groups"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "Logical backup export target must not be inside source"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "Logical backup exported from %s to %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "Dry run completed; no data was written"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "Logical backup import completed: %d event groups, checkpoint %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "Logical backup option --%s is required"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "Logical backup password environment variable is not set"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "Logical backup handshake failed: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "Logical backup request type %d failed: %s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = + "Use --input to specify the input. Export also requires --output; export can use --backup-dir to create, monitor, and clean up a snapshot Pipe automatically. Import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "Logical backup archive entry is unsafe: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "Logical backup archive exceeds safety limit"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "Logical backup export source must be a directory or manifest: %s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "Specify exactly one of --password-stdin and --password-env for logical backup import"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "No password was read from standard input"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "Logical backup checkpoint does not match the source or target: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "Logical backup checkpoint is invalid: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "Unsupported logical backup export format: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 = + "Logical backup export timeout and poll interval must be positive"; + public static final String + LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B = + "Logical backup export %s: state=%s, remaining=%s, estimated_seconds=%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 = + "Logical backup export pipe %s failed: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B = + "Logical backup export pipe %s timed out after %d seconds"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE = + "Logical backup option --%s must be an integer"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = + "Logical backup option --%s has invalid identifier: %s"; + + public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D = + "export is deprecated; use dump to generate a backup and pack to create a ZIP."; + public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE = + "import is deprecated; use restore."; + public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE = + "Logical backup: %s"; + public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 = + "Logical backup packed from %s to %s"; +public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E = + "--stop-sequence requires a stable source sequence capability; use --until with this server version."; + public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E = + "Object filters require --model tree or --model table; split mixed-model filtered dumps into separate backups."; + public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 = + "Table filters use database-regex/table-regex; exclusions must use .* for either database or table."; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 = + "Invalid logical backup dump option: --%s"; + public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 = + "Invalid or conflicting logical backup arguments."; + public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E = + "Only --conflict fail is supported; restore never implicitly overwrites or deletes target objects."; + public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F = + "The existing backup task does not match this backup identity or capture options."; + public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 = + "Another dump process is managing this backup."; + public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 = + "Backup %s, task %s, directory %s: %s. The task is retained; query SHOW PIPE or repeat dump with the same options and --resume append."; + public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 = + "The backup task has not produced sealed streams yet; resume the same task to finish sealing."; + public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D = + "An existing checkpoint requires --resume; --resume requires an existing checkpoint."; + public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB = + "The target timestamp precision does not match the backup."; + public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C = + "The target version or data model is incompatible with the backup."; + public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 = + "The checkpoint must be outside the input backup."; + public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 = + "Restoring authorization objects requires --include-auth true; inspect the backup or use --dry-run first."; + public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 = + "logical-backup [options]\n dump Connect to a source and seal a backup directory; does not create a ZIP.\n inspect Show backup identity, scope, streams, segments, counts and status.\n verify Check structure, checksums and semantics without changing the input.\n pack Verify and package an existing directory as ZIP; accepts no connection options.\n restore Preflight and replay a directory, root manifest.json or ZIP.\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous requires --until; --stop-sequence is reserved and currently rejected.\n Object filters require one model: Tree uses path lists; Table uses database-regex/table-regex.\n Tree config/schema capture accepts a single prefix/full path without exclusions.\n Table exclusions must select whole databases or whole table-name patterns.\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n Restoring authorization objects requires --include-auth true (legacy config is conservative).\n A target connection is required for replay; local dry-run needs no connection.\n Target preflight checks version, precision, authentication and handshake support;\n object conflicts and per-object privileges are checked during replay.\n\nDeprecated aliases: export --input DIR --output ZIP = pack;\n export with source connection options = dump then pack; import = restore.\nExit code: 0 success, 1 failure. Errors and deprecation notices go to stderr."; } diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index a260732ab554a..7887bf51b3651 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -96,4 +96,117 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "不支持输入时间格式 {}，"; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "请输入类似 yyyy-MM-dd\\ HH:mm:ss.SSS 或 yyyy-MM-dd'T'HH:mm:ss.SSS 的格式%n"; + // Pipe 逻辑备份工具 + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "逻辑备份命令执行失败：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "未知的逻辑备份命令：%s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "逻辑备份导出必须指定 --output"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "流 %s：%d 条记录，%d 个已提交事件组"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "逻辑备份校验通过：%d 个流，%d 条记录，%d 个已提交事件组"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "逻辑备份导出目标不能位于源目录内"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "逻辑备份已从 %s 导出到 %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "试运行完成，未写入数据"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "逻辑备份导入完成：%d 个事件组，checkpoint 为 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "逻辑备份选项 --%s 为必填项"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "未设置逻辑备份密码环境变量"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "逻辑备份握手失败：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "逻辑备份请求类型 %d 执行失败：%s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = + "使用 --input 指定输入。导出还需要 --output；导出也可使用 --backup-dir 自动创建、监控并清理快照 Pipe。导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "逻辑备份归档项路径不安全：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "逻辑备份归档超过安全限制"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "逻辑备份导出源必须是目录或 manifest：%s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "逻辑备份导入必须且只能指定 --password-stdin 或 --password-env 中的一项"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "未能从标准输入读取密码"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "逻辑备份 checkpoint 与源备份或目标实例不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "逻辑备份 checkpoint 无效：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "不支持的逻辑备份导出格式：%s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 = + "逻辑备份导出 timeout 和轮询间隔必须为正数"; + public static final String + LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B = + "逻辑备份导出 %s：状态=%s，剩余=%s，预计秒数=%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 = + "逻辑备份导出 pipe %s 失败：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B = + "逻辑备份导出 pipe %s 在 %d 秒后超时"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE = + "逻辑备份选项 --%s 必须是整数"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = + "逻辑备份选项 --%s 的标识符无效：%s"; + public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D = + "export 已弃用；请使用 dump 生成备份，使用 pack 创建 ZIP。"; + public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE = + "import 已弃用；请使用 restore。"; + public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE = + "逻辑备份：%s"; + public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 = + "已将逻辑备份从 %s 打包至 %s"; +public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E = + "--stop-sequence 需要源端提供稳定序号能力；当前服务端版本请使用 --until。"; + public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E = + "对象过滤需要 --model tree 或 --model table；请将混合模型过滤拆为独立备份。"; + public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 = + "Table 过滤使用数据库正则/表名正则；排除过滤中数据库或表名必须有一项为 .*。"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 = + "逻辑备份 dump 选项无效：--%s"; + public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 = + "逻辑备份参数无效或相互冲突。"; + public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E = + "当前仅支持 --conflict fail；restore 不会隐式覆盖或删除目标对象。"; + public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F = + "已有备份任务与本次备份身份或捕获选项不一致。"; + public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 = + "其他 dump 进程正在管理此备份。"; + public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 = + "备份 %s，任务 %s，目录 %s：%s。任务已保留；可使用 SHOW PIPE 查询，或使用相同选项和 --resume append 继续 dump。"; + public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 = + "备份任务尚未产出封存的流；请继续原任务以完成封存。"; + public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D = + "使用已有 checkpoint 必须指定 --resume；--resume 要求 checkpoint 已存在。"; + public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB = + "目标时间精度与备份不一致。"; + public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C = + "目标版本或数据模型与备份不兼容。"; + public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 = + "checkpoint 必须位于输入备份之外。"; + public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 = + "恢复权限对象必须显式指定 --include-auth true；请先 inspect 备份或执行 --dry-run。"; + public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 = + "logical-backup [选项]\n dump 连接源端并封存备份目录，不生成 ZIP。\n inspect 查看备份身份、范围、流、segment、计数和状态。\n verify 检查结构、校验和与语义，不修改输入。\n pack 校验已有目录并打包为 ZIP，不接受数据库连接参数。\n restore 预检并回放备份目录、根 manifest.json 或 ZIP。\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous 必须提供 --until；--stop-sequence 是预留参数，当前会被拒绝。\n 对象过滤只允许选择一个模型：Tree 使用路径列表，Table 使用数据库正则/表名正则。\n Tree config/schema 捕获只接受单个前缀或完整路径，不支持排除过滤。\n Table 排除过滤必须覆盖整个数据库或整个表名模式。\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n 恢复权限对象必须指定 --include-auth true；旧 config 备份按包含权限对象处理。\n 实际恢复需要目标连接；本地 dry-run 无需连接。\n 目标预检检查版本、时间精度、认证和握手能力；\n 对象冲突及对象级权限在回放时检查。\n\n弃用别名：export --input DIR --output ZIP = pack；\n 带源端连接参数的 export = dump 后接 pack；import = restore。\n退出码：0 成功，1 失败。错误及弃用提示输出到标准错误。"; } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java new file mode 100644 index 0000000000000..e28e288df0b42 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.cli.i18n.CliMessages; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import com.google.gson.Gson; +import org.apache.commons.cli.CommandLine; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** Converts the public dump contract to source and sink parameters before any remote mutation. */ +final class LogicalBackupDumpOptions { + + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + final Map source = new LinkedHashMap<>(); + final Map sink = new LinkedHashMap<>(); + final String resume; + final long until; + + LogicalBackupDumpOptions(final CommandLine line, final boolean legacy) throws IOException { + catalog.scope = + choices( + line.getOptionValue("scope", legacy ? "config,schema,data" : ""), + List.of("config", "schema", "data"), + "scope"); + catalog.model = + choices(line.getOptionValue("model", "tree,table"), List.of("tree", "table"), "model"); + catalog.mode = line.getOptionValue("mode", "snapshot"); + if (!List.of("snapshot", "continuous").contains(catalog.mode)) { + throw invalid("mode"); + } + final String includeAuth = line.getOptionValue("include-auth", "false"); + if (!List.of("true", "false").contains(includeAuth)) { + throw invalid("include-auth"); + } + catalog.includeAuth = Boolean.parseBoolean(includeAuth); + if (catalog.includeAuth && !catalog.scope.contains("config")) { + throw invalid("include-auth"); + } + if (line.hasOption("stop-sequence")) { + throw new IOException( + CliMessages + .EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E); + } + if ("continuous".equals(catalog.mode)) { + if (!line.hasOption("until")) { + throw invalid("until"); + } + try { + until = Instant.parse(line.getOptionValue("until")).toEpochMilli(); + } catch (final java.time.DateTimeException e) { + throw invalid("until"); + } + if (until <= System.currentTimeMillis() && !"append".equals(line.getOptionValue("resume"))) { + throw invalid("until"); + } + } else { + if (line.hasOption("until")) { + throw invalid("until"); + } + until = -1; + } + resume = line.getOptionValue("resume", "fail-if-exists"); + if (!List.of("fail-if-exists", "append", "new").contains(resume)) { + throw invalid("resume"); + } + catalog.captureOptions.put("scope", String.join(",", catalog.scope)); + catalog.captureOptions.put("model", String.join(",", catalog.model)); + catalog.captureOptions.put("mode", catalog.mode); + catalog.captureOptions.put("include-auth", includeAuth); + for (final String key : List.of("include", "exclude", "start-time", "end-time", "until")) { + if (line.hasOption(key)) { + catalog.filters.put(key, line.getOptionValue(key)); + catalog.captureOptions.put(key, line.getOptionValue(key)); + } + } + catalog.filters.put("time-boundary", "closed-closed"); + final List inclusion = new ArrayList<>(); + if (catalog.scope.contains("config")) { + inclusion.add("schema.database"); + inclusion.add("schema.ttl"); + inclusion.add("schema.timeseries.template.create"); + inclusion.add("schema.timeseries.template.set"); + inclusion.add("schema.timeseries.template.unset"); + inclusion.add("schema.timeseries.template.alter"); + inclusion.add("schema.timeseries.template.drop"); + } + if (catalog.scope.contains("schema")) { + inclusion.add("schema"); + } + if (catalog.scope.contains("data")) { + inclusion.add("data"); + } + if (catalog.includeAuth) { + inclusion.add("auth"); + } + source.put("source", "iotdb-source"); + source.put("source.inclusion", String.join(",", inclusion)); + source.put("source.inclusion.exclusion", catalog.includeAuth ? "" : "auth"); + source.put("source.skipif", ""); + source.put("source.capture.tree", Boolean.toString(catalog.model.contains("tree"))); + source.put("source.capture.table", Boolean.toString(catalog.model.contains("table"))); + source.put("source.mode", "snapshot".equals(catalog.mode) ? "snapshot" : "live"); + source.put("source.realtime.enable", Boolean.toString("continuous".equals(catalog.mode))); + source.put("source.logical-backup.snapshot", Boolean.toString("snapshot".equals(catalog.mode))); + if (until > 0) { + source.put("source.logical-backup.until", Long.toString(until)); + source.put("source.realtime.mode", "forced-log"); + } + // Decode TsFiles so the source applies exact path/time filters to logical rows. + source.put("source.mods.enable", "true"); + for (final String key : List.of("start-time", "end-time")) { + if (line.hasOption(key)) { + source.put("source." + key, line.getOptionValue(key)); + } + } + applyObjectFilters(line); + sink.put("sink", "logical-backup-sink"); + sink.put("sink.resume", "append"); + final Map defaults = new LinkedHashMap<>(); + defaults.put("segment-size-bytes", "268435456"); + defaults.put("max-record-bytes", "67108864"); + defaults.put("fsync-policy", "always"); + defaults.put("on-unsupported", "fail"); + defaults.put("include-heartbeat", "false"); + for (final Map.Entry entry : defaults.entrySet()) { + catalog.captureOptions.put( + entry.getKey(), line.getOptionValue(entry.getKey(), entry.getValue())); + sink.put( + "sink." + + ("on-unsupported".equals(entry.getKey()) ? "on-unsupported-event" : entry.getKey()), + catalog.captureOptions.get(entry.getKey())); + } + for (final String key : List.of("fsync-batch-operations", "fsync-period-ms")) { + if (line.hasOption(key)) { + sink.put("sink." + key, line.getOptionValue(key)); + catalog.captureOptions.put(key, line.getOptionValue(key)); + } + } + sink.put("sink.capture-options", new Gson().toJson(catalog.captureOptions)); + // Reuse the same validation as all three server-side writers. + final Map validation = new LinkedHashMap<>(sink); + validation.put("sink.dir", line.getOptionValue("backup-dir", ".")); + new LogicalBackupSettings(new PipeParameters(validation), "backup"); + } + + private void applyObjectFilters(final CommandLine line) throws IOException { + if (!line.hasOption("include") && !line.hasOption("exclude")) { + return; + } + if (catalog.model.size() != 1) { + throw new IOException( + CliMessages + .EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E); + } + if (catalog.model.contains("tree")) { + if (line.hasOption("include")) { + source.put("source.path.inclusion", line.getOptionValue("include")); + } + if (line.hasOption("exclude")) { + source.put("source.path.exclusion", line.getOptionValue("exclude")); + } + final TreePattern pattern = + TreePattern.parsePipePatternFromSourceParameters(new PipeParameters(source)); + if (!pattern.isLegal() + || ((catalog.scope.contains("config") || catalog.scope.contains("schema")) + && !(pattern instanceof IoTDBTreePatternOperations + && ((IoTDBTreePatternOperations) pattern).isPrefixOrFullPath()))) { + throw invalid("include/exclude"); + } + return; + } + // TablePattern accepts independent database/table regular expressions. Reject correlated + // exclusions rather than silently backing up their Cartesian product. + final String[] include = tableFilter(line.getOptionValue("include", ".*/.*")); + final String[] exclude = + line.hasOption("exclude") ? tableFilter(line.getOptionValue("exclude")) : null; + if (exclude != null && !".*".equals(exclude[0]) && !".*".equals(exclude[1])) { + throw new IOException( + CliMessages + .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23); + } + for (int index = 0; index < 2; index++) { + String regex = include[index]; + if (exclude != null && !".*".equals(exclude[index])) { + regex = "(?!(?:" + exclude[index] + ")$)(?:" + regex + ")"; + } + source.put(index == 0 ? "source.database-name" : "source.table-name", regex); + } + } + + private static String[] tableFilter(final String filter) throws IOException { + final String[] parts = filter.split("/", -1); + if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) { + throw new IOException( + CliMessages + .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23); + } + try { + Pattern.compile(parts[0]); + Pattern.compile(parts[1]); + } catch (final java.util.regex.PatternSyntaxException e) { + throw invalid("include/exclude"); + } + return parts; + } + + private static List choices( + final String value, final List allowed, final String key) throws IOException { + final List values = Arrays.asList(value.split(",", -1)); + if (values.isEmpty() + || !allowed.containsAll(values) + || values.stream().distinct().count() != values.size()) { + throw invalid(key); + } + // Canonical ordering makes equivalent requests compare equal when resuming. + final List canonical = new ArrayList<>(allowed); + canonical.retainAll(values); + return canonical; + } + + String createSql(final String pipeName, final String root, final String backupId) { + final Map sinkOptions = new LinkedHashMap<>(sink); + sinkOptions.put("sink.dir", root); + sinkOptions.put("sink.backup-id", backupId); + return "CREATE PIPE " + + pipeName + + " WITH SOURCE (" + + sqlOptions(source) + + ") WITH SINK (" + + sqlOptions(sinkOptions) + + ")"; + } + + private static String sqlOptions(final Map options) { + final List values = new ArrayList<>(); + options.forEach((key, value) -> values.add("'" + key + "'='" + value.replace("'", "''") + "'")); + return String.join(",", values); + } + + private static IOException invalid(final String key) { + return new IOException( + String.format(CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0, key)); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java new file mode 100644 index 0000000000000..e73067ed69ea9 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -0,0 +1,1705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.cli.i18n.CliMessages; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; +import org.apache.iotdb.session.Session; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.thrift.TException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */ +public final class PipeLogicalBackupTool { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final String COMMAND_DUMP = "dump"; + private static final String COMMAND_PACK = "pack"; + private static final String COMMAND_RESTORE = "restore"; + private static final String COMMAND_INSPECT = "inspect"; + private static final String COMMAND_VERIFY = "verify"; + private static final String COMMAND_EXPORT = "export"; + private static final String COMMAND_IMPORT = "import"; + private static final String OPTION_INPUT = "input"; + private static final String OPTION_OUTPUT = "output"; + private static final String OPTION_RESUME = "resume"; + private static final String OPTION_FORMAT = "format"; + private static final String OPTION_DEEP = "deep"; + // Compatibility aliases for the initial command-line prototype. + private static final String OPTION_SOURCE = "source"; + private static final String OPTION_TARGET = "target"; + private static final String OPTION_HOST = "host"; + private static final String OPTION_PORT = "port"; + private static final String OPTION_USER = "user"; + private static final String OPTION_PASSWORD_STDIN = "password-stdin"; + private static final String OPTION_PASSWORD_ENV = "password-env"; + private static final String OPTION_CHECKPOINT = "checkpoint"; + private static final String OPTION_DRY_RUN = "dry-run"; + private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete"; + private static final String OPTION_JSON = "json"; + private static final String OPTION_BACKUP_DIR = "backup-dir"; + private static final String OPTION_PIPE_NAME = "pipe-name"; + private static final String OPTION_BACKUP_ID = "backup-id"; + private static final String OPTION_TIMEOUT_SECONDS = "timeout-seconds"; + private static final String OPTION_POLL_INTERVAL_SECONDS = "poll-interval-seconds"; + private static final int MAX_ARCHIVE_ENTRIES = 1_000_000; + private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024; + private static final String CHECKPOINT_FORMAT_NAME = + "iotdb-pipe-logical-backup-import-checkpoint"; + private static final String CHECKPOINT_FORMAT_VERSION = "1.0"; + + private PipeLogicalBackupTool() {} + + public static void main(final String[] args) { + int exitCode = 0; + try { + exitCode = run(args); + } catch (final Exception e) { + System.err.println( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0, e.getMessage())); + exitCode = 1; + } + if (exitCode != 0) { + System.exit(exitCode); + } + } + + public static int run(final String[] args) throws Exception { + if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) { + printUsage(); + return 0; + } + final String command = args[0].toLowerCase(java.util.Locale.ROOT); + if (!List.of( + COMMAND_DUMP, + COMMAND_PACK, + COMMAND_RESTORE, + COMMAND_INSPECT, + COMMAND_VERIFY, + COMMAND_EXPORT, + COMMAND_IMPORT) + .contains(command)) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command)); + } + if (java.util.Arrays.asList(args).contains("--help") + || java.util.Arrays.asList(args).contains("-h")) { + printUsage(); + return 0; + } + final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length)); + switch (command) { + case COMMAND_INSPECT: + return inspect(line); + case COMMAND_VERIFY: + return verify(line); + case COMMAND_DUMP: + final Path dumped = dump(line, false); + printDumpResult(line, dumped); + return 0; + case COMMAND_PACK: + return pack(line); + case COMMAND_RESTORE: + return importBackup(line); + case COMMAND_EXPORT: + System.err.println( + CliMessages + .LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D); + if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) { + final Path source = dump(line, true); + final List streams = + new LogicalBackupArchiveReader() + .read(source, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return exportArchive( + source, + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)), + streams, + line.hasOption(OPTION_JSON)); + } + return pack(line); + case COMMAND_IMPORT: + System.err.println(CliMessages.LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE); + return importBackup(line); + default: + throw new AssertionError(command); + } + } + + static CommandLine parse(final String command, final String[] args) throws ParseException { + final Options options = new Options(); + final Set names = new LinkedHashSet<>(List.of(OPTION_JSON, OPTION_ALLOW_INCOMPLETE)); + final boolean legacyExport = COMMAND_EXPORT.equals(command); + final boolean legacyImport = COMMAND_IMPORT.equals(command); + final boolean dump = COMMAND_DUMP.equals(command) || legacyExport; + final boolean restore = COMMAND_RESTORE.equals(command) || legacyImport; + if (!COMMAND_DUMP.equals(command)) { + names.add(OPTION_INPUT); + } + if (COMMAND_PACK.equals(command) || legacyExport) { + names.add(OPTION_OUTPUT); + } + if (dump || restore) { + names.addAll( + List.of( + OPTION_HOST, + OPTION_PORT, + OPTION_USER, + OPTION_PASSWORD_STDIN, + OPTION_PASSWORD_ENV, + OPTION_RESUME)); + } + if (restore) { + names.addAll(List.of(OPTION_CHECKPOINT, OPTION_DRY_RUN, "conflict", "include-auth")); + } + if (dump) { + names.addAll( + List.of( + OPTION_BACKUP_DIR, + OPTION_BACKUP_ID, + OPTION_PIPE_NAME, + OPTION_TIMEOUT_SECONDS, + OPTION_POLL_INTERVAL_SECONDS, + "scope", + "model", + "include", + "exclude", + "start-time", + "end-time", + "include-auth", + "mode", + "until", + "stop-sequence", + "segment-size-bytes", + "max-record-bytes", + "fsync-policy", + "fsync-batch-operations", + "fsync-period-ms", + "on-unsupported", + "include-heartbeat")); + } + if (legacyExport || legacyImport) { + names.add(OPTION_SOURCE); + if (legacyExport) { + names.addAll(List.of(OPTION_TARGET, OPTION_FORMAT)); + } + } + if (COMMAND_VERIFY.equals(command)) { + names.add(OPTION_DEEP); // Accepted for compatibility; verification is always deep. + } + final Set flags = + Set.of( + OPTION_JSON, + OPTION_ALLOW_INCOMPLETE, + OPTION_DRY_RUN, + OPTION_PASSWORD_STDIN, + OPTION_DEEP); + for (final String name : names) { + if (OPTION_RESUME.equals(name) && restore) { + options.addOption( + Option.builder().longOpt(name).hasArg(legacyImport).optionalArg(legacyImport).build()); + } else { + addOption(options, name, !flags.contains(name), false); + } + } + final CommandLine line = new DefaultParser().parse(options, args); + if (!line.getArgList().isEmpty() + || (line.hasOption(OPTION_INPUT) && line.hasOption(OPTION_SOURCE)) + || (line.hasOption(OPTION_OUTPUT) && line.hasOption(OPTION_TARGET))) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + if (!COMMAND_DUMP.equals(command) + && !(legacyExport && line.hasOption(OPTION_BACKUP_DIR)) + && optionValue(line, OPTION_INPUT, OPTION_SOURCE) == null) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + if ((COMMAND_PACK.equals(command) || legacyExport) + && optionValue(line, OPTION_OUTPUT, OPTION_TARGET) == null) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_OUTPUT)); + } + if (legacyExport && !"binary".equals(line.getOptionValue(OPTION_FORMAT, "binary"))) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, + line.getOptionValue(OPTION_FORMAT))); + } + if (line.hasOption(OPTION_PORT)) { + final long port = longOption(line, OPTION_PORT, 6667); + if (port <= 0 || port > 65535) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + if (restore && !"fail".equals(line.getOptionValue("conflict", "fail"))) { + throw new ParseException( + CliMessages + .EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E); + } + // Offline export is exactly pack, including its refusal of source connection options. + if (legacyExport && optionValue(line, OPTION_INPUT, OPTION_SOURCE) != null) { + final Set packOptions = + Set.of( + OPTION_INPUT, + OPTION_SOURCE, + OPTION_OUTPUT, + OPTION_TARGET, + OPTION_FORMAT, + OPTION_JSON, + OPTION_ALLOW_INCOMPLETE); + for (final Option option : line.getOptions()) { + if (!packOptions.contains(option.getLongOpt())) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + } + return line; + } + + private static void addOption( + final Options options, final String name, final boolean hasArg, final boolean required) { + options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build()); + } + + private static Map summary(final List streams) { + final Map result = new LinkedHashMap<>(); + if (!streams.isEmpty() && streams.get(0).getCatalog() != null) { + result.put("manifest", streams.get(0).getCatalog()); + } + final List> details = new ArrayList<>(); + for (final BackupStream stream : streams) { + final Map item = new LinkedHashMap<>(); + item.put("manifest", stream.getManifest()); + item.put("recordCount", stream.getRecords().size()); + item.put("eventGroupCount", stream.getEventGroups().size()); + details.add(item); + } + result.put("streams", details); + return result; + } + + private static int inspect(final CommandLine line) throws IOException { + final List streams = read(line); + printSummary(line, summary(streams)); + return 0; + } + + private static int verify(final CommandLine line) throws IOException { + final List streams = read(line); + final Map result = summary(streams); + result.put("structural", "passed"); + result.put("checksum", "passed"); + result.put("semantic", "passed"); + printSummary(line, result); + return 0; + } + + private static void printSummary(final CommandLine line, final Map result) { + if (line.hasOption(OPTION_JSON)) { + System.out.println(GSON.toJson(result)); + } else { + System.out.println( + String.format(CliMessages.LOG_LOGICAL_BACKUP_ARG_1E5C23CE, GSON.toJson(result))); + } + } + + private static int pack(final CommandLine line) throws IOException { + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + final Path target = + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); + final List streams = read(line); + return exportArchive(source, target, streams, line.hasOption(OPTION_JSON)); + } + + private static Path dump(final CommandLine line, final boolean legacy) throws Exception { + final LogicalBackupDumpOptions options = new LogicalBackupDumpOptions(line, legacy); + final String host = required(line, OPTION_HOST); + final int port = Integer.parseInt(required(line, OPTION_PORT)); + final String user = line.getOptionValue(OPTION_USER, "root"); + final Path backupRoot = Path.of(required(line, OPTION_BACKUP_DIR)).toAbsolutePath().normalize(); + String backupId = + legacy + ? line.getOptionValue( + OPTION_BACKUP_ID, "backup_" + UUID.randomUUID().toString().replace('-', '_')) + : required(line, OPTION_BACKUP_ID); + LogicalBackupSettings.validateBackupId(backupId); + if ("new".equals(options.resume) + && Files.exists(backupRoot.resolve(backupId), LinkOption.NOFOLLOW_LINKS)) { + backupId += "-new-" + UUID.randomUUID().toString().substring(0, 8); + LogicalBackupSettings.validateBackupId(backupId); + } + final Path directory = backupRoot.resolve(backupId); + LogicalBackupCatalog.rejectSymbolicLinks(directory); + final boolean append = "append".equals(options.resume); + final boolean existed = Files.exists(directory, LinkOption.NOFOLLOW_LINKS); + if (existed && !append) { + throw new java.nio.file.FileAlreadyExistsException(directory.toString()); + } + final String password = password(line); + final long timeoutSeconds = longOption(line, OPTION_TIMEOUT_SECONDS, 3600L); + final long pollIntervalSeconds = longOption(line, OPTION_POLL_INTERVAL_SECONDS, 2L); + if (timeoutSeconds <= 0 + || timeoutSeconds > 31_536_000L + || pollIntervalSeconds <= 0 + || pollIntervalSeconds > 3600L) { + throw new IOException( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769); + } + LogicalBackupCatalog catalog = options.catalog; + catalog.backupId = backupId; + catalog.pipeName = + line.getOptionValue( + OPTION_PIPE_NAME, "logical_backup_" + UUID.randomUUID().toString().replace('-', '_')); + validateIdentifier(catalog.pipeName, OPTION_PIPE_NAME); + if (existed) { + final LogicalBackupCatalog previous = + LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + if (previous == null + || !catalog.backupId.equals(previous.backupId) + || !catalog.captureOptions.equals(previous.captureOptions) + || (line.hasOption(OPTION_PIPE_NAME) && !catalog.pipeName.equals(previous.pipeName))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F); + } + catalog = previous; + if ("SEALED".equals(catalog.status) || "INCOMPLETE".equals(catalog.status)) { + new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return directory; + } + } else { + Files.createDirectories(backupRoot); + Files.createDirectory(directory); + } + // Keep the lock inode: deleting it after unlock can let two later callers lock different files. + final Path lock = directory.resolve(".backup.lock"); + try (final java.nio.channels.FileChannel channel = + java.nio.channels.FileChannel.open( + lock, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + final java.nio.channels.FileLock ignored = channel.tryLock()) { + if (ignored == null) { + throw new IOException( + CliMessages.EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828); + } + catalog.write(directory); + final String pipeName = catalog.pipeName; + try (final Session session = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + session.open(); + final PipeStatus existing = readPipeStatus(session, pipeName); + if (existing != null && !append) { + throw new IOException( + CliMessages + .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F); + } + if (existing == null && (!existed || streamManifests(directory).isEmpty())) { + session.executeNonQueryStatement( + options.createSql(pipeName, backupRoot.toString(), backupId)); + session.executeNonQueryStatement("START PIPE " + pipeName); + } + waitForPipe( + session, + pipeName, + timeoutSeconds, + pollIntervalSeconds, + options.until, + line.hasOption(OPTION_JSON)); + final List streams = + awaitSealedStreams(directory, Math.min(timeoutSeconds, 120L), pollIntervalSeconds); + catalog.seal(directory, streams); + } catch (final Exception e) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87, + backupId, + pipeName, + directory, + e.getMessage()), + e); + } + } + new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return directory; + } + + private static List streamManifests(final Path root) throws IOException { + try (final java.util.stream.Stream paths = Files.walk(root)) { + return paths + .filter(path -> !path.equals(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME))) + .filter( + path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString())) + .sorted() + .toList(); + } + } + + private static List readGeneratedStreams(final Path root) throws Exception { + final List streams = new ArrayList<>(); + for (final Path manifest : streamManifests(root)) { + streams.addAll(new LogicalBackupArchiveReader().read(manifest, true)); + } + if (streams.isEmpty() + || streams.stream() + .anyMatch( + stream -> + stream.getManifest().segments.stream() + .anyMatch(segment -> !"SEALED".equals(segment.status)))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599); + } + return streams; + } + + private static List awaitSealedStreams( + final Path root, final long timeoutSeconds, final long pollSeconds) throws Exception { + final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L; + Exception lastFailure; + do { + try { + return readGeneratedStreams(root); + } catch (final IOException e) { + lastFailure = e; + } + Thread.sleep(Math.min(pollSeconds * 1000L, 1000L)); + } while (System.nanoTime() < deadline); + throw lastFailure; + } + + private static void printDumpResult(final CommandLine line, final Path directory) + throws IOException { + final LogicalBackupCatalog catalog = + LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + final Map result = new LinkedHashMap<>(); + result.put("backupId", catalog.backupId); + result.put("manifest", directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME).toString()); + result.put("status", catalog.status); + result.put("sourceHighWatermark", catalog.sourceHighWatermark); + printSummary(line, result); + } + + private static int exportArchive( + final Path source, final Path target, final List streams, final boolean json) + throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + final Path normalizedTarget = target.toAbsolutePath().normalize(); + final Path root; + if (Files.isDirectory(normalizedSource)) { + root = normalizedSource; + } else if (Files.isRegularFile(normalizedSource) + && "manifest.json".equals(normalizedSource.getFileName().toString())) { + root = normalizedSource.getParent(); + } else { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + normalizedSource)); + } + LogicalBackupCatalog.rejectSymbolicLinks(root); + LogicalBackupCatalog.rejectSymbolicLinks(normalizedTarget); + if (normalizedTarget.startsWith(root.toRealPath())) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994); + } + if (Files.exists(normalizedTarget, LinkOption.NOFOLLOW_LINKS)) { + throw new java.nio.file.FileAlreadyExistsException(normalizedTarget.toString()); + } + final Path parent = normalizedTarget.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp"); + final boolean incomplete = + streams.stream() + .anyMatch( + stream -> + stream.getManifest().skippedEventCount > 0 + || "INCOMPLETE".equals(stream.getManifest().status) + || (stream.getCatalog() != null + && "INCOMPLETE".equals(stream.getCatalog().status))); + boolean published = false; + try { + final boolean legacy = + LogicalBackupCatalog.load(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)) == null; + final boolean relocateLegacyStream = + legacy + && streams.stream() + .anyMatch(stream -> stream.getManifestPath().getParent().equals(root)); + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) { + if (legacy) { + final LogicalBackupManifest first = streams.get(0).getManifest(); + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = first.backupId; + catalog.pipeName = first.pipeName; + catalog.scope = + streams.stream().map(stream -> stream.getManifest().streamType).distinct().toList(); + catalog.model = List.of("tree", "table"); + catalog.includeAuth = containsConfigStream(streams); + catalog.captureOptions.putAll(first.captureOptions); + catalog.collect(root, streams); + if (relocateLegacyStream) { + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + entry.manifest.file = "legacy-stream/" + entry.manifest.file; + for (final LogicalBackupCatalog.FileEntry segment : entry.segments) { + segment.file = "legacy-stream/" + segment.file; + } + } + } + zip.putNextEntry(new ZipEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); + zip.write(GSON.toJson(catalog).getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + for (final Path file : exportFiles(root, streams)) { + final String entryName = + (relocateLegacyStream ? "legacy-stream/" : "") + + root.relativize(file).toString().replace('\\', '/'); + zip.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zip); + zip.closeEntry(); + } + } + try (final BackupInput packed = BackupInput.open(temporary)) { + new LogicalBackupArchiveReader().read(packed.getPath(), incomplete); + } + try (final java.nio.channels.FileChannel archiveChannel = + java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) { + archiveChannel.force(true); + } + try { + Files.move(temporary, normalizedTarget, StandardCopyOption.ATOMIC_MOVE); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, normalizedTarget); + } + published = true; + } finally { + if (!published) { + Files.deleteIfExists(temporary); + } + } + if (json) { + System.out.println( + GSON.toJson( + Map.of( + "input", + normalizedSource.toString(), + "output", + normalizedTarget.toString(), + "status", + incomplete ? "INCOMPLETE" : "SEALED"))); + } else { + System.out.println( + String.format( + CliMessages.LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304, + normalizedSource, + normalizedTarget)); + } + return 0; + } + + private static void waitForPipe( + final Session session, + final String pipeName, + final long timeoutSeconds, + final long pollSeconds, + final long until, + final boolean json) + throws Exception { + final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L; + while (System.nanoTime() < deadline) { + final PipeStatus status = readPipeStatus(session, pipeName); + if (status == null) { + return; + } else { + (json ? System.err : System.out) + .println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B, + pipeName, + status.state, + status.remainingEvents, + status.estimatedSeconds)); + if (status.state != null + && (status.state.equalsIgnoreCase("ERROR") + || status.state.equalsIgnoreCase("STOPPED"))) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190, + pipeName, + status.exceptionMessage)); + } + } + Thread.sleep(pollSeconds * 1000L); + } + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B, + pipeName, + timeoutSeconds)); + } + + private static PipeStatus readPipeStatus(final Session session, final String pipeName) + throws Exception { + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW PIPE " + pipeName)) { + final SessionDataSet.DataIterator iterator = dataSet.iterator(); + if (!iterator.next()) { + return null; + } + final PipeStatus status = new PipeStatus(); + status.state = value(iterator, dataSet, "State"); + status.remainingEvents = value(iterator, dataSet, "RemainingEventCount"); + status.estimatedSeconds = value(iterator, dataSet, "EstimatedRemainingSeconds"); + status.exceptionMessage = value(iterator, dataSet, "ExceptionMessage"); + return status; + } + } + + private static String value( + final SessionDataSet.DataIterator iterator, final SessionDataSet dataSet, final String name) + throws Exception { + for (final String column : dataSet.getColumnNames()) { + if (column.equalsIgnoreCase(name)) { + return iterator.getString(column); + } + } + return "Unknown"; + } + + private static long longOption(final CommandLine line, final String name, final long fallback) + throws ParseException { + try { + return Long.parseLong(line.getOptionValue(name, Long.toString(fallback))); + } catch (final NumberFormatException e) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE, name)); + } + } + + private static void validateIdentifier(final String value, final String option) + throws IOException { + if (!value.matches("[A-Za-z_][A-Za-z0-9_]{0,127}")) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204, + option, + value)); + } + } + + private static String sqlString(final String value) { + return value.replace("'", "''"); + } + + private static final class PipeStatus { + private String state; + private String remainingEvents; + private String estimatedSeconds; + private String exceptionMessage; + } + + private static List exportFiles(final Path root, final List streams) + throws IOException { + final Set files = new LinkedHashSet<>(); + final Path rootManifest = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (LogicalBackupCatalog.load(rootManifest) != null) { + files.add(rootManifest); + } + for (final BackupStream stream : streams) { + final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize(); + if (!manifestPath.startsWith(root)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + root)); + } + files.add(manifestPath); + final Path streamDirectory = manifestPath.getParent(); + for (final LogicalBackupManifest.Segment segment : stream.getManifest().segments) { + files.add(streamDirectory.resolve(segment.file).normalize()); + } + } + final List sortedFiles = new ArrayList<>(files); + sortedFiles.sort(Comparator.comparing(Path::toString)); + return sortedFiles; + } + + private static int importBackup(final CommandLine line) throws Exception { + final List streams = read(line); + final Map report = summary(streams); + final boolean authorizationObjects = + streams.get(0).getCatalog() == null + ? containsConfigStream(streams) + : streams.get(0).getCatalog().includeAuth; + final String includeAuth = line.getOptionValue("include-auth", "false"); + if (!List.of("true", "false").contains(includeAuth)) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0, + "include-auth")); + } + report.put("authorizationObjects", authorizationObjects); + report.put("authorizationRestoreSelected", Boolean.parseBoolean(includeAuth)); + if (authorizationObjects) { + report.put( + "authorizationSensitiveFields", + List.of("password-hashes", "role-memberships", "privileges")); + } + report.put("conflict", "fail"); + report.put( + "requestCount", + streams.stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .mapToLong(group -> group.getRequests().size()) + .sum()); + report.put("dryRun", line.hasOption(OPTION_DRY_RUN)); + final boolean targetProvided = + line.hasOption(OPTION_HOST) + || line.hasOption(OPTION_PORT) + || line.hasOption(OPTION_USER) + || line.hasOption(OPTION_PASSWORD_STDIN) + || line.hasOption(OPTION_PASSWORD_ENV); + if (!targetProvided && line.hasOption(OPTION_DRY_RUN)) { + report.put("targetPreflight", "not-requested"); + printSummary(line, report); + return 0; + } + if (authorizationObjects + && !Boolean.parseBoolean(includeAuth) + && !line.hasOption(OPTION_DRY_RUN)) { + throw new IOException( + CliMessages + .EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79); + } + final String host = required(line, OPTION_HOST); + final int port = Integer.parseInt(required(line, OPTION_PORT)); + final String user = line.getOptionValue(OPTION_USER, "root"); + final String password = password(line); + report.put("target", Map.of("host", host, "port", port, "user", user)); + final Path checkpoint = checkpointPath(line); + final boolean resume = line.hasOption(OPTION_RESUME); + if (!line.hasOption(OPTION_DRY_RUN) + && Files.exists(checkpoint, LinkOption.NOFOLLOW_LINKS) != resume) { + throw new IOException( + CliMessages + .EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D); + } + final ImportCheckpoint state = + line.hasOption(OPTION_DRY_RUN) + ? null + : readCheckpoint(checkpoint, streams, host, port, user); + if (state != null) { + final String sourceDigest = sourceDigest(Path.of(input(line)), streams); + if (state.sourceSha256 != null && !state.sourceSha256.equals(sourceDigest)) { + throw invalidCheckpoint(checkpoint); + } + state.sourceSha256 = sourceDigest; + } + try (final Session session = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + session.open(); + final String precision = session.getTimestampPrecision(); + if (!streams.get(0).getManifest().timestampPrecision.equals(precision)) { + throw new IOException( + CliMessages + .EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB); + } + String version = "unknown"; + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW VERSION")) { + final SessionDataSet.DataIterator iterator = dataSet.iterator(); + if (iterator.next()) { + version = iterator.getString(dataSet.getColumnNames().get(0)); + } + } + final LogicalBackupCatalog catalog = streams.get(0).getCatalog(); + final String sourceVersion = streams.get(0).getManifest().sourceVersion; + if ((sourceVersion != null && !version.split("\\.")[0].equals(sourceVersion.split("\\.")[0])) + || (catalog != null && catalog.model.contains("table") && version.startsWith("1."))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C); + } + report.put("targetVersion", version); + report.put("timestampPrecision", precision); + } + try (final IoTDBSyncClient client = + new IoTDBSyncClient( + new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(), + host, + port, + false, + "", + "")) { + preflight(client, streams, user, password); + report.put("targetPreflight", "passed"); + report.put("conflictCheck", "on-replay"); + if (line.hasOption(OPTION_DRY_RUN)) { + printSummary(line, report); + return 0; + } + (line.hasOption(OPTION_JSON) ? System.err : System.out).println(GSON.toJson(report)); + final java.util.concurrent.atomic.AtomicBoolean uncertainRequest = + new java.util.concurrent.atomic.AtomicBoolean( + state.inProgress != null + && state.inProgress.uncertain + && streams.stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .anyMatch( + group -> + group + .getEventGroupId() + .toString() + .equals(state.inProgress.eventGroupId) + && state.inProgress.nextRequestIndex + < group.getRequests().size())); + try { + state.status = "RUNNING"; + state.lastError = null; + long importedGroups = 0; + for (final BackupStream stream : streams) { + importedGroups += + importStream( + checkpoint, + state, + stream, + request -> transfer(client, request, uncertainRequest.getAndSet(false)), + checkpointState -> writeCheckpoint(checkpoint, checkpointState)); + } + state.status = "COMPLETED"; + writeCheckpoint(checkpoint, state); + report.put("status", state.status); + report.put("importedEventGroups", importedGroups); + report.put("checkpoint", checkpoint.toString()); + printSummary(line, report); + } catch (final Exception e) { + state.status = "FAILED"; + state.lastError = e.getClass().getSimpleName(); + try { + writeCheckpoint(checkpoint, state); + } catch (final IOException checkpointFailure) { + e.addSuppressed(checkpointFailure); + } + throw e; + } + } + return 0; + } + + static void preflight( + final IoTDBSyncClient client, + final List streams, + final String user, + final String password) + throws IOException, TException { + handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + if (containsConfigStream(streams)) { + configNodeHandshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + } + } + + private static String sourceDigest(final Path input, final List streams) + throws IOException { + if (Files.isRegularFile(input) + && !LogicalBackupFormat.MANIFEST_FILE_NAME.equals(input.getFileName().toString())) { + return LogicalBackupCatalog.sha256(input); + } + final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, GSON.toJson(sourceStreamIdentities(streams))); + if (!streams.isEmpty() && streams.get(0).getCatalog() != null) { + updateDigest(digest, GSON.toJson(streams.get(0).getCatalog())); + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + static boolean containsConfigStream(final List streams) { + return streams.stream() + .anyMatch(stream -> "config".equalsIgnoreCase(stream.getManifest().streamType)); + } + + static long importStream( + final Path checkpoint, + final ImportCheckpoint checkpointState, + final BackupStream stream, + final RequestTransfer requestTransfer, + final CheckpointPersister checkpointPersister) + throws Exception { + final String streamId = stream.getManifest().streamId; + final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); + long importedGroups = 0; + for (final EventGroup group : stream.getEventGroups()) { + if (group.getLastSequence() <= lastApplied) { + continue; + } + + int nextRequestIndex = 0; + if (checkpointState.inProgress == null) { + checkpointState.inProgress = ImportProgress.start(streamId, group); + checkpointPersister.persist(checkpointState); + } else if (checkpointState.inProgress.matches(streamId, group)) { + nextRequestIndex = checkpointState.inProgress.nextRequestIndex; + } else { + throw invalidCheckpoint(checkpoint); + } + + final List requests = group.getRequests(); + if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) { + throw invalidCheckpoint(checkpoint); + } + for (int index = nextRequestIndex; index < requests.size(); index++) { + checkpointState.inProgress.uncertain = true; + checkpointPersister.persist(checkpointState); + try { + requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + } catch (final RestoreRequestRejectedException e) { + checkpointState.inProgress.uncertain = false; + throw e; + } + checkpointState.inProgress.nextRequestIndex = index + 1; + checkpointState.inProgress.uncertain = false; + checkpointPersister.persist(checkpointState); + } + + checkpointState.appliedSequences.put(streamId, group.getLastSequence()); + checkpointState.inProgress = null; + checkpointPersister.persist(checkpointState); + importedGroups++; + } + return importedGroups; + } + + private static List read(final CommandLine line) throws IOException { + final String source = input(line); + try (final BackupInput input = BackupInput.open(Path.of(source))) { + return new LogicalBackupArchiveReader() + .read(input.getPath(), line.hasOption(OPTION_ALLOW_INCOMPLETE)); + } + } + + private static String input(final CommandLine line) throws IOException { + final String value = optionValue(line, OPTION_INPUT, OPTION_SOURCE); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + return value; + } + + private static String optionValue( + final CommandLine line, final String preferredOption, final String compatibilityOption) { + return line.hasOption(preferredOption) + ? line.getOptionValue(preferredOption) + : line.getOptionValue(compatibilityOption); + } + + private static String required(final CommandLine line, final String option) throws IOException { + final String value = line.getOptionValue(option); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, option)); + } + return value; + } + + private static String password(final CommandLine line) throws IOException { + final boolean fromStandardInput = line.hasOption(OPTION_PASSWORD_STDIN); + final boolean fromEnvironment = line.hasOption(OPTION_PASSWORD_ENV); + if (fromStandardInput == fromEnvironment) { + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + if (fromStandardInput) { + final BufferedReader reader = + new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + final String value = reader.readLine(); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E); + } + return value; + } + if (fromEnvironment) { + final String value = System.getenv(line.getOptionValue(OPTION_PASSWORD_ENV)); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2); + } + return value; + } + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + + private static void handshake( + final IoTDBSyncClient client, + final String timestampPrecision, + final String user, + final String password) + throws IOException, TException { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"); + final TPipeTransferResp response = + client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params)); + if (response == null + || response.getStatus() == null + || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697, + response == null ? "null" : response.getStatus())); + } + } + + private static void configNodeHandshake( + final IoTDBSyncClient client, + final String timestampPrecision, + final String user, + final String password) + throws IOException, TException { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"); + final TPipeTransferResp response = + client.pipeTransfer( + PipeTransferHandshakeV2Req.toTPipeTransferReq( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params)); + if (response == null + || response.getStatus() == null + || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697, + response == null ? "null" : response.getStatus())); + } + } + + static void transfer( + final IoTDBSyncClient client, + final TPipeTransferReq request, + final boolean allowIdempotentReplay) + throws TException, IOException { + final TPipeTransferResp response = client.pipeTransfer(request); + if (response == null + || response.getStatus() == null + || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() + && (!allowIdempotentReplay + || response.getStatus().getCode() + != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))) { + throw new RestoreRequestRejectedException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, + request.getType(), + response == null ? "null" : response.getStatus())); + } + } + + private static Path checkpointPath(final CommandLine line) throws IOException { + final String configuredCheckpoint = + line.hasOption(OPTION_CHECKPOINT) + ? line.getOptionValue(OPTION_CHECKPOINT) + : line.getOptionValue(OPTION_RESUME); + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + final Path checkpoint = + (configuredCheckpoint == null + ? source.resolveSibling(source.getFileName() + ".restore.checkpoint.json") + : Path.of(configuredCheckpoint)) + .toAbsolutePath() + .normalize(); + LogicalBackupCatalog.rejectSymbolicLinks(checkpoint); + final Path sourceRoot = + Files.isDirectory(source) + ? source + : LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString()) + ? source.getParent() + : source; + if (checkpoint.startsWith(sourceRoot)) { + throw new IOException( + CliMessages.EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59); + } + return checkpoint; + } + + static ImportCheckpoint readCheckpoint( + final Path checkpoint, + final List streams, + final String host, + final int port, + final String user) + throws IOException { + final Map sourceStreams = sourceStreamIdentities(streams); + if (!Files.exists(checkpoint)) { + final ImportCheckpoint state = new ImportCheckpoint(); + state.formatName = CHECKPOINT_FORMAT_NAME; + state.formatVersion = CHECKPOINT_FORMAT_VERSION; + state.sourceStreams = sourceStreams; + state.backupId = streams.get(0).getManifest().backupId; + state.backupFormatVersion = streams.get(0).getManifest().formatVersion; + state.targetHost = host; + state.targetPort = port; + state.targetUser = user; + return state; + } + final ImportCheckpoint state; + try { + state = + GSON.fromJson( + Files.readString(checkpoint, StandardCharsets.UTF_8), ImportCheckpoint.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint), + e); + } + if (state == null + || !CHECKPOINT_FORMAT_NAME.equals(state.formatName) + || !CHECKPOINT_FORMAT_VERSION.equals(state.formatVersion) + || state.sourceStreams == null + || state.appliedSequences == null) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); + } + if (!streams.get(0).getManifest().backupId.equals(state.backupId) + || !streams.get(0).getManifest().formatVersion.equals(state.backupFormatVersion) + || !sourceStreams.equals(state.sourceStreams) + || !host.equals(state.targetHost) + || port != state.targetPort + || !user.equals(state.targetUser) + || !checkpointStateIsValid(streams, state)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D, + checkpoint)); + } + return state; + } + + static Map sourceStreamIdentities(final List streams) { + final Map identities = new LinkedHashMap<>(); + for (final BackupStream stream : streams) { + final LogicalBackupManifest manifest = stream.getManifest(); + final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, manifest.formatName); + updateDigest(digest, manifest.formatVersion); + updateDigest(digest, manifest.timestampPrecision); + updateDigest(digest, manifest.sourceVersion); + updateDigest(digest, GSON.toJson(manifest.captureOptions)); + updateDigest(digest, manifest.backupId); + updateDigest(digest, manifest.streamId); + updateDigest(digest, manifest.pipeName); + updateDigest(digest, manifest.pipeCreationTime); + updateDigest(digest, manifest.sourceClusterId); + updateDigest(digest, manifest.regionId); + updateDigest(digest, manifest.streamType); + updateDigest(digest, manifest.segments.size()); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + updateDigest(digest, segment.file); + updateDigest(digest, segment.segmentId); + updateDigest(digest, segment.firstSequence); + updateDigest(digest, segment.lastSequence); + updateDigest(digest, segment.recordCount); + updateDigest(digest, segment.sizeBytes); + updateDigest(digest, segment.sha256); + updateDigest(digest, segment.status); + } + updateDigest(digest, stream.getRecords().size()); + for (final LogicalBackupRecord record : stream.getRecords()) { + digest.update(record.getRecordType().getCode()); + digest.update( + ByteBuffer.allocate(Long.BYTES * 4 + Integer.BYTES + Byte.BYTES + Short.BYTES) + .putLong(record.getSequence()) + .putLong(record.getEventGroupId().getMostSignificantBits()) + .putLong(record.getEventGroupId().getLeastSignificantBits()) + .putInt(record.getOperationIndex()) + .putLong(record.getEventTime()) + .put(record.getRequestVersion()) + .putShort(record.getRequestType()) + .array()); + updateDigest(digest, record.getMetadata()); + updateDigest(digest, record.getPayload()); + } + identities.put(manifest.streamId, LogicalBackupFormat.toHex(digest.digest())); + } + return identities; + } + + private static void updateDigest(final MessageDigest digest, final String value) { + if (value == null) { + digest.update((byte) 0); + return; + } + digest.update((byte) 1); + updateDigest(digest, value.getBytes(StandardCharsets.UTF_8)); + } + + private static void updateDigest(final MessageDigest digest, final byte[] value) { + updateDigest(digest, value.length); + digest.update(value); + } + + private static void updateDigest(final MessageDigest digest, final int value) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } + + private static void updateDigest(final MessageDigest digest, final long value) { + digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array()); + } + + private static boolean checkpointStateIsValid( + final List streams, final ImportCheckpoint state) { + final Map appliedSequences = state.appliedSequences; + if (appliedSequences.size() > streams.size()) { + return false; + } + boolean foundIncompleteStream = false; + boolean matchedInProgress = false; + for (final BackupStream stream : streams) { + final String streamId = stream.getManifest().streamId; + final Long applied = appliedSequences.getOrDefault(streamId, -1L); + if (applied == null || applied < -1) { + return false; + } + + int appliedGroupIndex = -1; + if (applied != -1) { + for (int index = 0; index < stream.getEventGroups().size(); index++) { + if (stream.getEventGroups().get(index).getLastSequence() == applied) { + appliedGroupIndex = index; + break; + } + } + if (appliedGroupIndex < 0) { + return false; + } + } + + final boolean streamComplete = appliedGroupIndex == stream.getEventGroups().size() - 1; + if (foundIncompleteStream && appliedGroupIndex >= 0) { + return false; + } + if (!streamComplete && !foundIncompleteStream) { + foundIncompleteStream = true; + if (state.inProgress != null) { + final EventGroup nextGroup = stream.getEventGroups().get(appliedGroupIndex + 1); + if (!state.inProgress.matches(streamId, nextGroup) + || state.inProgress.nextRequestIndex < 0 + || state.inProgress.nextRequestIndex > nextGroup.getRequests().size()) { + return false; + } + matchedInProgress = true; + } + } + } + return (state.inProgress == null || matchedInProgress) + && appliedSequences.keySet().stream() + .allMatch( + streamId -> + streams.stream() + .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + } + + private static IOException invalidCheckpoint(final Path checkpoint) { + return new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); + } + + static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) + throws IOException { + final Path parent = checkpoint.toAbsolutePath().normalize().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + LogicalBackupCatalog.rejectSymbolicLinks(checkpoint); + checkpointState.updatedAt = java.time.Instant.now().toString(); + final Path temporary = Files.createTempFile(parent, ".checkpoint-", ".tmp"); + final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState)); + try (final java.nio.channels.FileChannel checkpointChannel = + java.nio.channels.FileChannel.open( + temporary, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { + while (checkpointBytes.hasRemaining()) { + checkpointChannel.write(checkpointBytes); + } + checkpointChannel.force(true); + } + try { + Files.move( + temporary, + checkpoint, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static String getClusterId() { + final byte[] bytes = new byte[16]; + new SecureRandom().nextBytes(bytes); + return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes); + } + + private static void printUsage() { + System.out.println( + CliMessages + .LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8); + } + + static class ImportCheckpoint { + String formatName; + String formatVersion; + String sourceSha256; + String backupId; + String backupFormatVersion; + String status = "RUNNING"; + String updatedAt; + String lastError; + Map sourceStreams = new LinkedHashMap<>(); + String targetHost; + int targetPort; + String targetUser; + Map appliedSequences = new LinkedHashMap<>(); + ImportProgress inProgress; + } + + static class ImportProgress { + String streamId; + String eventGroupId; + long lastSequence; + int nextRequestIndex; + boolean uncertain; + + static ImportProgress start(final String streamId, final EventGroup group) { + final ImportProgress progress = new ImportProgress(); + progress.streamId = streamId; + progress.eventGroupId = group.getEventGroupId().toString(); + progress.lastSequence = group.getLastSequence(); + return progress; + } + + boolean matches(final String candidateStreamId, final EventGroup group) { + return candidateStreamId.equals(streamId) + && group.getEventGroupId().toString().equals(eventGroupId) + && group.getLastSequence() == lastSequence; + } + } + + private static class RestoreRequestRejectedException extends IOException { + private RestoreRequestRejectedException(final String message) { + super(message); + } + } + + @FunctionalInterface + interface RequestTransfer { + void transfer(TPipeTransferReq request) throws Exception; + } + + @FunctionalInterface + interface CheckpointPersister { + void persist(ImportCheckpoint state) throws IOException; + } + + private static class BackupInput implements AutoCloseable { + private final Path path; + private final Path temporaryDirectory; + + private BackupInput(final Path path, final Path temporaryDirectory) { + this.path = path; + this.temporaryDirectory = temporaryDirectory; + } + + private static BackupInput open(final Path source) throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + if (!Files.isRegularFile(normalizedSource) + || "manifest.json".equals(normalizedSource.getFileName().toString())) { + return new BackupInput(normalizedSource, null); + } + final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-"); + try { + extract(normalizedSource, temporaryDirectory); + return new BackupInput(temporaryDirectory, temporaryDirectory); + } catch (final IOException | RuntimeException e) { + deleteRecursively(temporaryDirectory); + if (e instanceof IOException) { + throw (IOException) e; + } + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e); + } + } + + private static void extract(final Path archive, final Path target) throws IOException { + int entryCount = 0; + long totalBytes = 0; + final byte[] buffer = new byte[64 * 1024]; + try (final ZipFile zip = new ZipFile(archive.toFile())) { + final Map entries = new LinkedHashMap<>(); + final Set allowed = new LinkedHashSet<>(); + final java.util.Enumeration enumeration = zip.entries(); + while (enumeration.hasMoreElements()) { + final ZipEntry entry = enumeration.nextElement(); + if (++entryCount > MAX_ARCHIVE_ENTRIES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + LogicalBackupCatalog.resolve( + target, + entry.isDirectory() + ? entry.getName().substring(0, entry.getName().length() - 1) + : entry.getName()); + if (entries.putIfAbsent(entry.getName(), entry) != null + || entry.getSize() < 0 + || entry.getSize() > MAX_ARCHIVE_BYTES + || (entry.getSize() > 1024 * 1024 + && entry.getSize() / Math.max(1, entry.getCompressedSize()) > 1000)) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + } + final ZipEntry rootManifest = entries.get(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (rootManifest != null) { + final com.google.gson.JsonObject rootJson = manifestJson(zip, rootManifest); + if (rootJson.has("streams")) { + final LogicalBackupCatalog catalog = + GSON.fromJson(rootJson, LogicalBackupCatalog.class); + allowed.add(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (catalog.streams == null) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + for (final LogicalBackupCatalog.StreamEntry stream : catalog.streams) { + if (stream == null + || stream.manifest == null + || stream.segments == null + || !allowed.add(stream.manifest.file)) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + for (final LogicalBackupCatalog.FileEntry segment : stream.segments) { + if (segment == null || !allowed.add(segment.file)) { + throw new IOException( + CliMessages + .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + } + } + } + if (allowed.isEmpty()) { + // Read archives produced by the initial stream-only format with the same allowlist. + for (final ZipEntry entry : entries.values()) { + if (!entry.isDirectory() && entry.getName().endsWith("manifest.json")) { + final LogicalBackupManifest manifest = + GSON.fromJson(manifestJson(zip, entry), LogicalBackupManifest.class); + if (manifest.segments == null || !allowed.add(entry.getName())) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + final Path parent = Path.of(entry.getName()).getParent(); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + if (segment == null + || segment.file == null + || !allowed.add( + parent == null + ? segment.file + : parent.resolve(segment.file).toString().replace('\\', '/'))) { + throw new IOException( + CliMessages + .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + } + } + } + if (allowed.isEmpty()) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + for (final String file : allowed) { + LogicalBackupCatalog.resolve(target, file); + if (!entries.containsKey(file) || entries.get(file).isDirectory()) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + for (final ZipEntry entry : entries.values()) { + if (entry.isDirectory()) { + if (allowed.stream().noneMatch(file -> file.startsWith(entry.getName()))) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + continue; + } + if (!allowed.contains(entry.getName())) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + final Path output = LogicalBackupCatalog.resolve(target, entry.getName()); + Files.createDirectories(output.getParent()); + try (final java.io.InputStream input = zip.getInputStream(entry); + final OutputStream stream = + Files.newOutputStream( + output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + int length; + while ((length = input.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + totalBytes += length; + if (totalBytes > MAX_ARCHIVE_BYTES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + stream.write(buffer, 0, length); + } + } + } + } + } + + private static com.google.gson.JsonObject manifestJson(final ZipFile zip, final ZipEntry entry) + throws IOException { + if (entry.isDirectory() || entry.getSize() > 16 * 1024 * 1024) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + try (final java.io.InputStream input = zip.getInputStream(entry)) { + final byte[] bytes = input.readNBytes(16 * 1024 * 1024 + 1); + final com.google.gson.JsonObject json = + GSON.fromJson( + new String(bytes, StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + if (bytes.length > 16 * 1024 * 1024 || json == null) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + return json; + } catch (final JsonParseException | IllegalStateException e) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e); + } + } + + private Path getPath() { + return path; + } + + @Override + public void close() throws IOException { + if (temporaryDirectory != null) { + deleteRecursively(temporaryDirectory); + } + } + + private static void deleteRecursively(final Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (final java.util.stream.Stream paths = Files.walk(directory)) { + for (final Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java new file mode 100644 index 0000000000000..4fee78c4b15b0 --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -0,0 +1,520 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.commons.cli.ParseException; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +public class PipeLogicalBackupToolTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testDirectoryPackZipVerifyAndRestoreDryRun() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source); + + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run(new String[] {"verify", "--input", source.toString(), "--deep"})); + + final Path archive = root.resolve("backup.zip"); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()})); + Assert.assertTrue(Files.isRegularFile(archive)); + try (final ZipFile zip = new ZipFile(archive.toFile())) { + Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); + Assert.assertNotNull(zip.getEntry("legacy-stream/manifest.json")); + Assert.assertNotNull(zip.getEntry("legacy-stream/segment-00000000000000000000.pwal")); + Assert.assertNull(zip.getEntry(".backup.lock")); + Assert.assertEquals(3, zip.size()); + } + + Assert.assertEquals( + 0, PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"restore", "--input", archive.toString(), "--dry-run"})); + + final byte[] originalArchive = Files.readAllBytes(archive); + try { + PipeLogicalBackupTool.run( + new String[] {"export", "--input", source.toString(), "--output", archive.toString()}); + Assert.fail(); + } catch (final java.nio.file.FileAlreadyExistsException expected) { + Assert.assertArrayEquals(originalArchive, Files.readAllBytes(archive)); + } + } + + @Test + public void testPackPreservesIncompleteStatus() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("incomplete"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "data-1"; + manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + source, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { + writer.recordSkippedEvent(1, "unsupported event"); + } + final Path archive = root.resolve("incomplete.zip"); + try { + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()}); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertFalse(Files.exists(archive)); + } + final PrintStream previous = System.out; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run( + new String[] { + "pack", + "--input", + source.toString(), + "--output", + archive.toString(), + "--allow-incomplete", + "--json" + }); + final com.google.gson.JsonObject json = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals("INCOMPLETE", json.get("status").getAsString()); + bytes.reset(); + PipeLogicalBackupTool.run( + new String[] {"verify", "--input", archive.toString(), "--allow-incomplete", "--json"}); + final com.google.gson.JsonObject verified = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals( + "INCOMPLETE", verified.getAsJsonObject("manifest").get("status").getAsString()); + } finally { + System.setOut(previous); + } + } + + @Test + public void testCheckpointSourceIdentityIncludesContent() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Map before = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + streams.get(0).getManifest().segments.get(0).sha256 = "different-content"; + final Map after = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + Assert.assertNotEquals(before, after); + } + + @Test + public void testConfigStreamDetection() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path data = root.resolve("data"); + final Path config = root.resolve("config"); + writeBackup(data); + writeBackup(config, "config-0", "config", Collections.singletonList(request((short) 200))); + + Assert.assertFalse( + PipeLogicalBackupTool.containsConfigStream( + new LogicalBackupArchiveReader().read(data, false))); + Assert.assertTrue( + PipeLogicalBackupTool.containsConfigStream( + new LogicalBackupArchiveReader().read(config, false))); + } + + @Test + public void testZipSlipIsRejected() throws Exception { + final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip"); + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive))) { + zip.putNextEntry(new ZipEntry("../outside")); + zip.write(1); + zip.closeEntry(); + } + + try { + PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()}); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + @Test + public void testPlaintextPasswordOptionIsRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + try { + PipeLogicalBackupTool.run( + new String[] { + "import", "--source", source.toString(), "--dry-run", "--password", "secret" + }); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + + @Test + public void testUnknownCommandsAreRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + for (final String command : Arrays.asList("stats", "generate")) { + try { + PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()}); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + } + + @Test + public void testRootManifestPackAndJsonVerification() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("backup"); + writeBackup(source.resolve("data-1")); + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = "backup"; + catalog.pipeName = "pipe"; + catalog.scope = Collections.singletonList("data"); + catalog.model = Collections.singletonList("tree"); + catalog.seal(source, new LogicalBackupArchiveReader().read(source, false)); + final byte[] originalManifest = Files.readAllBytes(source.resolve("manifest.json")); + final Path archive = root.resolve("rooted.zip"); + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()}); + Assert.assertArrayEquals(originalManifest, Files.readAllBytes(source.resolve("manifest.json"))); + try (final ZipFile zip = new ZipFile(archive.toFile())) { + Assert.assertNotNull(zip.getEntry("manifest.json")); + Assert.assertNotNull(zip.getEntry("data-1/manifest.json")); + Assert.assertEquals(3, zip.size()); + } + final PrintStream previous = System.out; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString(), "--json"}); + final com.google.gson.JsonObject json = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals("passed", json.get("checksum").getAsString()); + Assert.assertEquals("backup", json.getAsJsonObject("manifest").get("backupId").getAsString()); + } finally { + System.setOut(previous); + } + } + + @Test + public void testDeprecatedAliasesReportMigration() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source); + final PrintStream previous = System.err; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setErr(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run( + new String[] { + "export", + "--input", + source.toString(), + "--output", + root.resolve("legacy.zip").toString() + }); + PipeLogicalBackupTool.run(new String[] {"import", "--input", source.toString(), "--dry-run"}); + } finally { + System.setErr(previous); + } + final String notices = bytes.toString(StandardCharsets.UTF_8); + Assert.assertTrue(notices.contains("dump")); + Assert.assertTrue(notices.contains("pack")); + Assert.assertTrue(notices.contains("restore")); + } + + @Test + public void testCommandsRejectOptionsWithOtherResponsibilities() throws Exception { + final List invalid = + Arrays.asList( + new String[] { + "pack", "--input", "backup", "--output", "backup.zip", "--host", "localhost" + }, + new String[] {"dump", "--input", "backup"}, + new String[] {"dump", "--output", "backup.zip"}, + new String[] {"inspect", "--input", "backup", "--password-stdin"}, + new String[] {"restore", "--input", "backup", "--resume", "checkpoint.json"}, + new String[] {"restore", "--input", "backup", "--conflict", "overwrite"}); + for (final String[] args : invalid) { + try { + PipeLogicalBackupTool.run(args); + Assert.fail(Arrays.toString(args)); + } catch (final ParseException expected) { + // Command-specific validation must fail before opening files or connections. + } + } + } + + @Test + public void testDumpScopeAuthAndWriterOptionsAreExplicit() throws Exception { + final LogicalBackupDumpOptions options = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", + new String[] { + "--scope", + "schema,data", + "--model", + "tree", + "--include", + "root.a.**", + "--fsync-policy", + "periodic", + "--fsync-period-ms", + "42" + }), + false); + Assert.assertEquals("schema,data", options.source.get("source.inclusion")); + Assert.assertEquals("auth", options.source.get("source.inclusion.exclusion")); + Assert.assertEquals("false", options.source.get("source.capture.table")); + Assert.assertEquals("root.a.**", options.source.get("source.path.inclusion")); + Assert.assertEquals("42", options.sink.get("sink.fsync-period-ms")); + Assert.assertFalse(options.catalog.includeAuth); + final LogicalBackupDumpOptions auth = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", new String[] {"--scope", "config", "--include-auth", "true"}), + false); + Assert.assertTrue(auth.source.get("source.inclusion").contains("auth")); + } + + @Test + public void testDumpRejectsAmbiguousOrUnboundedCaptureBeforeCreatingTask() throws Exception { + for (final String[] args : + Arrays.asList( + new String[] {}, + new String[] {"--scope", "data", "--mode", "continuous"}, + new String[] {"--scope", "data", "--include-auth", "true"}, + new String[] {"--scope", "data", "--model", "tree,table", "--include", "root.a.**"}, + new String[] {"--scope", "data", "--segment-size-bytes", "1024"}, + new String[] {"--scope", "data", "--fsync-period-ms", "1000"}, + new String[] {"--scope", "data", "--mode", "continuous", "--stop-sequence", "100"})) { + try { + new LogicalBackupDumpOptions(PipeLogicalBackupTool.parse("dump", args), false); + Assert.fail(Arrays.toString(args)); + } catch (final IOException expected) { + // No unsupported option may broaden capture or leave an unbounded task. + } + } + } + + @Test + public void testPreflightSendsOnlyHandshakeRequests() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source, "config-0", "config", Collections.singletonList(request((short) 200))); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + Mockito.when(client.pipeTransfer(Mockito.any())) + .thenReturn( + new TPipeTransferResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + PipeLogicalBackupTool.preflight(client, streams, "root", "secret"); + final org.mockito.ArgumentCaptor requests = + org.mockito.ArgumentCaptor.forClass(TPipeTransferReq.class); + Mockito.verify(client, Mockito.times(2)).pipeTransfer(requests.capture()); + Assert.assertEquals( + PipeRequestType.HANDSHAKE_DATANODE_V2.getType(), requests.getAllValues().get(0).getType()); + Assert.assertEquals( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2.getType(), + requests.getAllValues().get(1).getType()); + } + + @Test + public void testFreshRestoreRejectsIdempotentConflict() throws Exception { + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + Mockito.when(client.pipeTransfer(Mockito.any())) + .thenReturn( + new TPipeTransferResp( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); + try { + PipeLogicalBackupTool.transfer(client, request(1), false); + Assert.fail(); + } catch (final IOException expected) { + // Only the uncertain request at a durable resume point may accept an idempotent conflict. + } + } + + @Test + public void testTransferAcceptsIdempotentConflict() throws Exception { + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + final TPipeTransferReq request = request(1); + Mockito.when(client.pipeTransfer(request)) + .thenReturn( + new TPipeTransferResp( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); + + PipeLogicalBackupTool.transfer(client, request, true); + } + + @Test + public void testImportCheckpointResumesAtRequestGranularity() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source, Arrays.asList(request(1), request(2))); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final BackupStream stream = streams.get(0); + final Path checkpoint = root.resolve("import.checkpoint.json"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final List transferredBodies = new ArrayList<>(); + final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true); + + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> { + if (checkpointState.inProgress != null + && checkpointState.inProgress.nextRequestIndex == 1 + && failCheckpointOnce.getAndSet(false)) { + throw new IOException("simulated checkpoint failure"); + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); + }); + Assert.fail(); + } catch (final IOException expected) { + // The request succeeded, but advancing its checkpoint was interrupted. + } + + Assert.assertEquals(Collections.singletonList(1), transferredBodies); + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertNotNull(durableState.inProgress); + Assert.assertEquals(0, durableState.inProgress.nextRequestIndex); + + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies); + Assert.assertNull(durableState.inProgress); + Assert.assertEquals( + Long.valueOf(stream.getEventGroups().get(0).getLastSequence()), + durableState.appliedSequences.get(stream.getManifest().streamId)); + } + + private static void writeBackup(final Path directory) throws Exception { + writeBackup(directory, Collections.singletonList(request(3))); + } + + private static void writeBackup(final Path directory, final List requests) + throws Exception { + writeBackup(directory, "data-1", "data", requests); + } + + private static void writeBackup( + final Path directory, + final String streamId, + final String streamType, + final List requests) + throws Exception { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = streamId; + manifest.streamType = streamType; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { + writer.writeEvent(UUID.randomUUID(), 1, requests, "metadata"); + } + } + + private static TPipeTransferReq request(final int value) { + return request((short) 10, value); + } + + private static TPipeTransferReq request(final short type) { + return request(type, 1); + } + + private static TPipeTransferReq request(final short type, final int value) { + return new TPipeTransferReq() + .setVersion((byte) 1) + .setType(type) + .setBody(ByteBuffer.wrap(new byte[] {(byte) value})); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java index c7222e9a1bc97..a0823225547b8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionAirGapSink; import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionSink; +import org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.pipe.api.PipeConnector; class PipeConfigRegionSinkConstructor extends PipeSinkConstructor { @@ -46,6 +47,7 @@ protected void initConstructors() { IoTDBConfigRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put("logical-backup-connector", LogicalBackupSink::new); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBConfigRegionSink::new); @@ -59,6 +61,7 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBConfigRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put("logical-backup-sink", LogicalBackupSink::new); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java index a6b1e95ae3cc2..f5726de92053b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; @@ -32,13 +33,18 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager; +import org.apache.iotdb.confignode.manager.pipe.source.ConfigRegionListeningFilter; import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -185,12 +191,21 @@ private void parseHeartbeatAndSaveMetaChangeLocally( // pipes must remain alive because they may receive work in the future. final boolean isFiniteInternalPipe = !staticMeta.isSourceExternal() - && PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters()); + && (PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters()) + || (staticMeta + .getSourceParameters() + .getLongOrDefault("source.logical-backup.until", -1L) + > 0 + && System.currentTimeMillis() + >= staticMeta + .getSourceParameters() + .getLongOrDefault("source.logical-backup.until", -1L))); final boolean hasReliableDataRegionReport = pipeHeartbeat.hasCompletedDataRegionReport(staticMeta); if (isFiniteInternalPipe && hasReliableDataRegionReport - && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)) { + && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds) + && metadataSnapshotsCompleted(pipeMetaFromCoordinator)) { PipeLogger.log( LOGGER::info, ManagerMessages.ALL_DATANODES_REPORTED_HISTORICAL_PIPE_COMPLETED, @@ -323,6 +338,54 @@ private Set collectRequiredDataRegionIds(final PipeMeta pipeMetaFromCoo return requiredDataRegionIds; } + private boolean metadataSnapshotsCompleted(final PipeMeta pipeMeta) { + final PipeParameters parameters = pipeMeta.getStaticMeta().getSourceParameters(); + if (!parameters.getBooleanOrDefault("source.logical-backup.snapshot", false) + && parameters.getLongOrDefault("source.logical-backup.until", -1L) <= 0) { + return true; + } + try { + for (final Map.Entry entry : + pipeMeta.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().entrySet()) { + final boolean required; + if (entry.getKey() == Integer.MIN_VALUE) { + required = !ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters).isEmpty(); + } else { + final TConsensusGroupId region = + new TConsensusGroupId(TConsensusGroupType.SchemaRegion, entry.getKey()); + if (!configManager.getPartitionManager().isRegionGroupExists(region) + || !SchemaRegionListeningFilter.shouldSchemaRegionBeListened(parameters)) { + continue; + } + final String database = configManager.getPartitionManager().getRegionDatabase(region); + final TDatabaseSchema schema = + configManager.getClusterSchemaManager().getDatabaseSchemaByName(database); + required = + schema.isIsTableModel() + ? TablePattern.isTableModelDataAllowToBeCaptured(parameters) + : TreePattern.isTreeModelDataAllowToBeCaptured(parameters); + } + if (required + && entry + .getValue() + .getProgressIndex() + .getProgressIndexByType(MetaProgressIndex.class) + .map( + index -> + index.getIndex() + != org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource + .LOGICAL_BACKUP_COMPLETED_INDEX) + .orElse(true)) { + return false; + } + } + return true; + } catch (final Exception e) { + // A failed metadata lookup must never turn a partial snapshot into a completed backup. + return false; + } + } + private boolean shouldKeepDataRegionAsRequired( final PipeStaticMeta staticMeta, final TConsensusGroupId dataRegionId) { if (staticMeta.isSourceExternal()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..789953a05d4da --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionSnapshotEvent; +import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionWritePlanEvent; +import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigPlanReq; +import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotPieceReq; +import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotSealReq; +import org.apache.iotdb.confignode.service.ConfigNode; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; +import org.apache.iotdb.pipe.api.PipeConnector; +import org.apache.iotdb.pipe.api.annotation.TableModel; +import org.apache.iotdb.pipe.api.annotation.TreeModel; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.File; +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@TreeModel +@TableModel +public class LogicalBackupSink implements PipeConnector { + private static final int FILE_PIECE_BYTES = 64 * 1024; + private LogicalBackupWriter writer; + private LogicalBackupSettings settings; + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + validator.validateSynonymAttributes( + Collections.singletonList("connector.dir"), Collections.singletonList("sink.dir"), true); + final LogicalBackupSettings validated = + new LogicalBackupSettings(validator.getParameters(), "backup"); + validated.requireSharedResume(validator.getParameters()); + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + final String pipeName = configuration.getRuntimeEnvironment().getPipeName(); + final long creationTime = configuration.getRuntimeEnvironment().getCreationTime(); + final int regionId = configuration.getRuntimeEnvironment().getRegionId(); + settings = new LogicalBackupSettings(parameters, pipeName + "-" + creationTime); + settings.requireSharedResume(parameters); + if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) { + return; + } + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = settings.backupId; + manifest.pipeName = pipeName; + manifest.pipeCreationTime = creationTime; + manifest.sourceClusterId = + ConfigNode.getInstance().getConfigManager().getClusterManager().getClusterId(); + manifest.sourceVersion = IoTDBConstant.VERSION; + manifest.timestampPrecision = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + manifest.streamId = pipeName + "-" + creationTime + "-config-" + regionId; + manifest.regionId = regionId; + manifest.streamType = "config"; + writer = settings.open("config", regionId, manifest); + } + + @Override + public void transfer(final TabletInsertionEvent event) throws Exception { + unsupported(event); + } + + @Override + public void transfer(final TsFileInsertionEvent event) throws Exception { + unsupported(event); + } + + @Override + public void transfer(final Event event) throws Exception { + if (event instanceof PipeConfigRegionWritePlanEvent) { + final PipeConfigRegionWritePlanEvent e = (PipeConfigRegionWritePlanEvent) event; + if (e.increaseReferenceCount(getClass().getName())) { + try { + writer.writeEvent( + eventId(e), + System.currentTimeMillis(), + Collections.singletonList( + PipeTransferConfigPlanReq.toTPipeTransferReq(e.getConfigPhysicalPlan())), + "config-plan"); + } finally { + e.decreaseReferenceCount(getClass().getName(), false); + } + } + } else if (event instanceof PipeConfigRegionSnapshotEvent) { + final PipeConfigRegionSnapshotEvent e = (PipeConfigRegionSnapshotEvent) event; + if (e.increaseReferenceCount(getClass().getName())) { + try { + final List requests = new ArrayList<>(); + appendFile(requests, e.getSnapshotFile()); + appendFile(requests, e.getTemplateFile()); + final File snapshot = e.getSnapshotFile(); + final File template = e.getTemplateFile(); + requests.add( + PipeTransferConfigSnapshotSealReq.toTPipeTransferReq( + e.getTreePattern().getPattern(), + e.getTablePattern().getDatabasePattern(), + e.getTablePattern().getTablePattern(), + e.getTreePattern().isTreeModelDataAllowedToBeCaptured(), + e.getTablePattern().isTableModelDataAllowedToBeCaptured(), + snapshot.getName(), + snapshot.length(), + template == null ? null : template.getName(), + template == null ? 0 : template.length(), + e.getFileType(), + e.toSealTypeString(), + e.getAuthUserName())); + writer.writeEvent(eventId(e), System.currentTimeMillis(), requests, "config-snapshot"); + } finally { + e.decreaseReferenceCount(getClass().getName(), false); + } + } + } else if (event instanceof PipeHeartbeatEvent) { + if (settings.includeHeartbeat) { + writer.writeControl( + LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), "heartbeat"); + } + } else if (event instanceof PipeTerminateEvent) { + // close() writes STREAM_END after every preceding request has been committed. + } else { + unsupported(event); + } + } + + private void unsupported(final Event event) throws Exception { + final String name = event == null ? "null" : event.getClass().getName(); + if ("skip".equals(settings.unsupported)) { + writer.recordSkippedEvent(System.currentTimeMillis(), name); + } else { + throw new java.io.IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60, name)); + } + } + + private static UUID eventId(final EnrichedEvent event) { + String identity = + event.getClass().getName() + + "\0" + + event.getPipeName() + + "\0" + + event.getCreationTime() + + "\0" + + event.getCommitId(); + if (event instanceof PipeConfigRegionSnapshotEvent) { + final PipeConfigRegionSnapshotEvent snapshot = (PipeConfigRegionSnapshotEvent) event; + identity += "\0" + snapshot.getFileType() + "\0" + snapshot.getSnapshotFile().getName(); + } + return UUID.nameUUIDFromBytes(identity.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + private static void appendFile(final List requests, final File file) + throws Exception { + if (file == null) return; + try (FileInputStream input = new FileInputStream(file)) { + final byte[] buffer = new byte[FILE_PIECE_BYTES]; + long offset = 0; + int length; + while ((length = input.read(buffer)) > 0) { + requests.add( + PipeTransferConfigSnapshotPieceReq.toTPipeTransferReq( + file.getName(), offset, Arrays.copyOf(buffer, length))); + offset += length; + } + } + } + + @Override + public void handshake() {} + + @Override + public void heartbeat() throws Exception { + if (writer != null) writer.heartbeat(); + } + + @Override + public void close() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java new file mode 100644 index 0000000000000..3031e2c6b216d --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.agent.plugin; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup.LogicalBackupSink; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class PipeConfigRegionSinkConstructorTest { + + @Test + public void testLogicalBackupAliasesUseConfigRegionSink() { + final PipeConfigRegionSinkConstructor constructor = new PipeConfigRegionSinkConstructor(); + + Assert.assertTrue( + constructor.reflectPlugin( + new PipeParameters( + Collections.singletonMap( + PipeSinkConstant.CONNECTOR_KEY, "logical-backup-connector"))) + instanceof LogicalBackupSink); + Assert.assertTrue( + constructor.reflectPlugin( + new PipeParameters( + Collections.singletonMap(PipeSinkConstant.SINK_KEY, "logical-backup-sink"))) + instanceof LogicalBackupSink); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java index ef4c03549afc7..7df84bd15820e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TPipeCompletedDataRegion; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; @@ -77,6 +78,37 @@ public class PipeHeartbeatParserTest { private boolean originalSeparatedPipeHeartbeatEnabled; + @Test + public void testBackupWaitsForCommittedConfigSnapshotAfterDataCompletes() throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + final Map source = new HashMap<>(); + source.put("source.realtime.enable", "false"); + source.put("source.logical-backup.snapshot", "true"); + source.put("source.inclusion", "schema.database"); + final PipeMeta meta = createPipeMeta(source, Integer.MIN_VALUE); + final PipeTaskInfo info = new PipeTaskInfo(); + info.createPipe(new CreatePipePlanV2(meta.getStaticMeta(), meta.getRuntimeMeta())); + final ParserTestContext context = createParserTestContext(1, info); + context.parser.parseHeartbeat( + 1, + createPipeHeartbeatWithCompletedRegions( + meta, true, Collections.singletonList(Integer.MIN_VALUE))); + Assert.assertNotNull(info.getPipeMetaByPipeName("test_pipe")); + info.getPipeMetaByPipeName("test_pipe") + .getRuntimeMeta() + .getConsensusGroupId2TaskMetaMap() + .get(Integer.MIN_VALUE) + .updateProgressIndex( + new MetaProgressIndex( + org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource + .LOGICAL_BACKUP_COMPLETED_INDEX)); + context.parser.parseHeartbeat( + 1, + createPipeHeartbeatWithCompletedRegions( + meta, true, Collections.singletonList(Integer.MIN_VALUE))); + Assert.assertNull(info.getPipeMetaByPipeName("test_pipe")); + } + @Before public void setUp() { originalSeparatedPipeHeartbeatEnabled = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java index bd77d72d1bef1..e8a17e5d524d6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigNodeHandshakeV1Req; @@ -93,12 +94,18 @@ public void testPipeTransferConfigHandshakeV2Req() throws IOException { final PipeTransferConfigNodeHandshakeV2Req req = PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferReq(params); + final TPipeTransferReq commonReq = + PipeTransferHandshakeV2Req.toTPipeTransferReq( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params); final int originalBodyPosition = req.body.position(); final PipeTransferConfigNodeHandshakeV2Req deserializeReq = PipeTransferConfigNodeHandshakeV2Req.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); Assert.assertEquals(req.getType(), deserializeReq.getType()); + Assert.assertEquals(req.getVersion(), commonReq.getVersion()); + Assert.assertEquals(req.getType(), commonReq.getType()); + Assert.assertArrayEquals(req.getBody(), commonReq.getBody()); Assert.assertEquals(originalBodyPosition, req.body.position()); Assert.assertEquals(params, deserializeReq.getParams()); } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index e676d5c93e94c..49ed04f7b75cd 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2706,6 +2706,20 @@ private DataNodePipeMessages() {} "Interrupted while reading OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "Failed to read OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "Unsupported logical backup event policy: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "Unsupported logical backup resume policy: %s"; + public static final String + EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B = + "ConfigRegion and DataRegion logical backup require explicit sink.resume=append (or connector.resume=append) and a shared backup directory"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "Unsupported Pipe event for logical backup: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Logical backup ID must resolve below the configured backup directory: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Logical backup directory must not be empty"; public static final String MESSAGE_ARG_SUBSCRIPTION_ENTRY_REQUIRES_ARG_BYTES_ABOVE_THE_CURRENT_PER_QUEUE_MAXIMUM_ARG_BYTES_DATANODE_BUDGET_ARG_BYTES_REDUCE_THE_WRITE_BATCH_FIELD_SIZE_OR_INCREASE_SUBSCRIPTION_MATERIALIZATION_MEMORY_WAL_PROGRESS_HAS_NOT_ADVANCED_AFCBC7FC = "[%s] Subscription entry requires %d bytes, above the current per-queue maximum %d bytes (DataNode budget %d bytes). Reduce the write batch/field size or increase subscription materialization memory. WAL progress has not advanced."; public static final String LOG_SUBSCRIPTION_FAILED_TO_REPLAY_DELIVERY_BATCH_ARG_FROM_WAL_RETAINING_UNCOMMITTED_STATE_0654EACD = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index a3949cd283e3b..2d668b059ce40 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2531,6 +2531,20 @@ private DataNodePipeMessages() {} "读取 OPC UA 服务器操作限制时被中断，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "读取 OPC UA 服务器操作限制失败，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "不支持的逻辑备份事件策略：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "不支持的逻辑备份恢复策略：%s"; + public static final String + EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B = + "ConfigRegion 和 DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "逻辑备份不支持 Pipe 事件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Pipe 逻辑备份 ID 必须解析到配置的备份目录下：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Pipe 逻辑备份目录不能为空"; public static final String MESSAGE_ARG_SUBSCRIPTION_ENTRY_REQUIRES_ARG_BYTES_ABOVE_THE_CURRENT_PER_QUEUE_MAXIMUM_ARG_BYTES_DATANODE_BUDGET_ARG_BYTES_REDUCE_THE_WRITE_BATCH_FIELD_SIZE_OR_INCREASE_SUBSCRIPTION_MATERIALIZATION_MEMORY_WAL_PROGRESS_HAS_NOT_ADVANCED_AFCBC7FC = "[%s] 订阅 entry 需要 %d 字节，超过当前队列可用上限 %d 字节（DataNode 预算 %d 字节）。请减小写入 batch/字段大小或增加订阅物化内存。WAL 进度未推进。"; public static final String LOG_SUBSCRIPTION_FAILED_TO_REPLAY_DELIVERY_BATCH_ARG_FROM_WAL_RETAINING_UNCOMMITTED_STATE_0654EACD = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java index 7044336f6397d..74693448da9f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink; @@ -72,6 +73,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_CONNECTOR.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBDataRegionAsyncSink::new); @@ -93,6 +97,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_SINK.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.SUBSCRIPTION_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java index 160ecf54c0158..a4eba6400c178 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink; import org.apache.iotdb.pipe.api.PipeConnector; @@ -46,6 +47,9 @@ protected void initConstructors() { IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("schema")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new); @@ -59,6 +63,9 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("schema")); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java index 7ac1f7ba12a45..ef404d4e5f8d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java @@ -81,12 +81,13 @@ public synchronized String register( connectorKey); final boolean isDataRegionSink = isDataRegionSink(environment.getRegionId()); + final boolean isLogicalBackupSink = isLogicalBackupSink(connectorKey); final int sinkNum = calculateSinkSubtaskNum(pipeSinkParameters, isDataRegionSink, connectorKey); boolean realTimeFirst = false; final String attributeSortedString = generateAttributeSortedString(pipeSinkParameters, environment.getRegionId()); final String attributeDisplayString = generateAttributeDisplayString(pipeSinkParameters); - if (isDataRegionSink) { + if (isDataRegionSink && !isLogicalBackupSink) { realTimeFirst = pipeSinkParameters.getBooleanOrDefault( Arrays.asList( @@ -95,11 +96,16 @@ public synchronized String register( PipeSinkConstant.CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE); } final String attributeDisplayStringWithPrefix = - isDataRegionSink - ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) - ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString - : "data_" + attributeDisplayString - : "schema_" + attributeDisplayString; + isLogicalBackupSink + ? (isDataRegionSink ? "data_region_" : "schema_region_") + + environment.getRegionId() + + "_" + + attributeDisplayString + : isDataRegionSink + ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) + ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString + : "data_" + attributeDisplayString + : "schema_" + attributeDisplayString; environment.setAttributeSortedString(attributeDisplayStringWithPrefix); final PipeSinkSubtaskKey pipeSinkSubtaskKey = new PipeSinkSubtaskKey( @@ -386,6 +392,13 @@ public static int calculateSinkSubtaskNum( public static String generateAttributeSortedString( final PipeParameters pipeSinkParameters, final int regionId) { final String attributeSortedString = generateAttributeSortedString(pipeSinkParameters); + if (isLogicalBackupSink(getConnectorKey(pipeSinkParameters))) { + // A backup stream belongs to exactly one region and must have one ordered writer. + return (isDataRegionSink(regionId) ? "data_region_" : "schema_region_") + + regionId + + "_" + + attributeSortedString; + } if (isDataRegionSink(regionId)) { return PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) ? "data_region_" + regionId + "_" + attributeSortedString @@ -398,6 +411,11 @@ private static String getConnectorKey(final PipeParameters pipeSinkParameters) { return PipeSinkConstant.getConnectorOrSinkNameWithDefault(pipeSinkParameters).toLowerCase(); } + private static boolean isLogicalBackupSink(final String connectorKey) { + return BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName().equals(connectorKey) + || BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName().equals(connectorKey); + } + private static boolean isDataRegionSink(final int regionId) { return StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId)) || PipeRuntimeMeta.isSourceExternal(regionId); @@ -407,7 +425,7 @@ private static int calculateSinkSubtaskNum( final PipeParameters pipeSinkParameters, final boolean isDataRegionSink, final String connectorKey) { - if (!isDataRegionSink) { + if (!isDataRegionSink || isLogicalBackupSink(connectorKey)) { // Do not allow parallel tasks for schema region connectors to avoid the potential disorder of // the schema region data transfer. return 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..a1ef953f4a007 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.pipe.api.PipeConnector; +import org.apache.iotdb.pipe.api.annotation.TableModel; +import org.apache.iotdb.pipe.api.annotation.TreeModel; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.write.record.Tablet; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; + +@TreeModel +@TableModel +public class LogicalBackupSink implements PipeConnector { + + private static final int FILE_PIECE_BYTES = 64 * 1024; + protected LogicalBackupWriter writer; + private String pipeName; + private long creationTime; + private int regionId; + protected boolean includeHeartbeat; + protected String unsupportedEventPolicy; + private final String streamType; + private long backupStartTime = Long.MIN_VALUE; + private long backupEndTime = Long.MAX_VALUE; + + public LogicalBackupSink() { + this("unknown"); + } + + public LogicalBackupSink(final String streamType) { + this.streamType = streamType; + } + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + final PipeParameters parameters = validator.getParameters(); + validator.validateSynonymAttributes( + Collections.singletonList(PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY), + Collections.singletonList(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY), + true); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validateSharedDirectoryResumePolicy(parameters); + new LogicalBackupSettings(parameters, "backup"); + final String policy = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + if (!PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL.equals(policy) + && !PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(policy)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405, + policy)); + } + } + + private static void validateSynonymAttributes( + final PipeParameterValidator validator, + final String connectorAttribute, + final String sinkAttribute) { + validator.validateSynonymAttributes( + Collections.singletonList(connectorAttribute), + Collections.singletonList(sinkAttribute), + false); + } + + private void validateSharedDirectoryResumePolicy(final PipeParameters parameters) + throws PipeParameterNotValidException { + if (!"data".equals(streamType) && !"config".equals(streamType)) { + return; + } + final boolean explicitlyConfigured = + parameters.hasAnyAttributes( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + final String resume = + parameters.getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), + PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE); + if (!explicitlyConfigured + || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) { + throw new PipeParameterNotValidException( + DataNodePipeMessages + .EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B); + } + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + pipeName = configuration.getRuntimeEnvironment().getPipeName(); + creationTime = configuration.getRuntimeEnvironment().getCreationTime(); + regionId = configuration.getRuntimeEnvironment().getRegionId(); + validateSharedDirectoryResumePolicy(parameters); + final LogicalBackupSettings settings = + new LogicalBackupSettings(parameters, pipeName + "-" + creationTime); + includeHeartbeat = settings.includeHeartbeat; + unsupportedEventPolicy = settings.unsupported; + if (settings.captureOptions.containsKey("start-time")) { + backupStartTime = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + settings.captureOptions.get("start-time")); + } + if (settings.captureOptions.containsKey("end-time")) { + backupEndTime = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + settings.captureOptions.get("end-time")); + } + if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) { + return; + } + final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId; + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = settings.backupId; + manifest.pipeName = pipeName; + manifest.pipeCreationTime = creationTime; + manifest.sourceClusterId = getSourceClusterId(); + manifest.sourceVersion = IoTDBConstant.VERSION; + manifest.timestampPrecision = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + manifest.streamId = streamId; + manifest.regionId = regionId; + manifest.streamType = streamType; + manifest.sinkTaskId = streamId; + writer = settings.open(streamType, regionId, manifest); + } + + protected String getSourceClusterId() { + return IoTDBDescriptor.getInstance().getConfig().getClusterId(); + } + + @Override + public void handshake() {} + + @Override + public void heartbeat() throws Exception { + if (writer != null) { + writer.heartbeat(); + } + } + + @Override + public void transfer(final TabletInsertionEvent event) throws Exception { + if (event instanceof PipeInsertNodeTabletInsertionEvent) { + final PipeInsertNodeTabletInsertionEvent insert = (PipeInsertNodeTabletInsertionEvent) event; + transferEnriched(insert, () -> requests(insert)); + } else if (event instanceof PipeRawTabletInsertionEvent) { + final PipeRawTabletInsertionEvent raw = (PipeRawTabletInsertionEvent) event; + transferEnriched( + raw, + () -> + Collections.singletonList( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + raw.convertToTablet(), + raw.isAligned(), + raw.isTableModelEvent() + ? raw.getTableModelDatabaseName() + : raw.getTreeModelDatabaseName()))); + } else { + handleUnsupported(event); + } + } + + @Override + public void transfer(final TsFileInsertionEvent event) throws Exception { + if (!(event instanceof PipeTsFileInsertionEvent)) { + handleUnsupported(event); + return; + } + final PipeTsFileInsertionEvent tsFile = (PipeTsFileInsertionEvent) event; + final List requests = new ArrayList<>(); + if (!tsFile.increaseReferenceCount(getClass().getName())) { + return; + } + try { + tsFile.consumeTabletInsertionEventsWithRetry( + tablet -> { + try { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablet.convertToTablet(), + tablet.isAligned(), + tablet.isTableModelEvent() + ? tablet.getTableModelDatabaseName() + : tablet.getTreeModelDatabaseName())); + } catch (final IOException e) { + throw new PipeException(e.getMessage(), e); + } + }, + getClass().getName()); + if (!requests.isEmpty()) { + writer.writeEvent(eventId(tsFile), System.currentTimeMillis(), requests, metadata(tsFile)); + } + } finally { + event.close(); + tsFile.decreaseReferenceCount(getClass().getName(), false); + } + } + + @Override + public void transfer(final Event event) throws Exception { + if (event instanceof PipeDeleteDataNodeEvent) { + final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event; + transferEnriched( + deletion, + () -> { + final Optional node = + boundedDeletion(deletion.getDeleteDataNode(), backupStartTime, backupEndTime); + return node.isPresent() + ? Collections.singletonList(PipeTransferPlanNodeReq.toTPipeTransferReq(node.get())) + : Collections.emptyList(); + }); + } else if (event instanceof PipeSchemaRegionWritePlanEvent) { + final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event; + transferEnriched( + schema, + () -> + Collections.singletonList( + PipeTransferPlanNodeReq.toTPipeTransferReq(schema.getPlanNode()))); + } else if (event instanceof PipeSchemaRegionSnapshotEvent) { + transferSnapshot((PipeSchemaRegionSnapshotEvent) event); + } else if (event instanceof PipeHeartbeatEvent) { + if (includeHeartbeat) { + writer.writeControl( + LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event)); + } + } else if (event instanceof PipeTerminateEvent) { + // The framework commits this ordered completion signal. close() writes the final STREAM_END; + // a continuous source also emits a historical completion signal before its realtime events. + } else { + handleUnsupported(event); + } + } + + private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + final List requests = new ArrayList<>(); + final File[] files = + new File[] { + event.getMTreeSnapshotFile(), + event.getTagLogSnapshotFile(), + event.getAttributeSnapshotFile() + }; + for (final File file : files) { + if (file == null) { + continue; + } + try (final FileInputStream input = new FileInputStream(file)) { + final byte[] piece = new byte[FILE_PIECE_BYTES]; + long offset = 0; + int length; + while ((length = input.read(piece)) >= 0) { + if (length == 0) { + continue; + } + requests.add( + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq( + file.getName(), offset, Arrays.copyOf(piece, length))); + offset += length; + } + } + } + requests.add( + PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( + event.getTreePattern().getPattern(), + event.getTablePattern().getDatabasePattern(), + event.getTablePattern().getTablePattern(), + event.getTreePattern().isTreeModelDataAllowedToBeCaptured(), + event.getTablePattern().isTableModelDataAllowedToBeCaptured(), + files[0] == null ? null : files[0].getName(), + files[0] == null ? 0 : files[0].length(), + files[1] == null ? null : files[1].getName(), + files[1] == null ? 0 : files[1].length(), + files[2] == null ? null : files[2].getName(), + files[2] == null ? 0 : files[2].length(), + event.getDatabaseName(), + event.toSealTypeString())); + writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event)); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + List requests(final PipeInsertNodeTabletInsertionEvent event) + throws IOException { + final String databaseName = + event.isTableModelEvent() + ? event.getTableModelDatabaseName() + : event.getTreeModelDatabaseName(); + if (!(event.getInsertNode() instanceof RelationalInsertTabletNode) + && !event.shouldParseTimeOrPattern()) { + return Collections.singletonList( + PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq( + event.getInsertNode(), databaseName)); + } + + final List tablets = event.convertToTablets(); + final List requests = new ArrayList<>(tablets.size()); + for (int index = 0; index < tablets.size(); index++) { + if (tablets.get(index).getRowSize() == 0) { + continue; + } + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablets.get(index), event.isAligned(index), databaseName)); + } + return requests; + } + + static Optional boundedDeletion( + final AbstractDeleteDataNode node, final long start, final long end) { + if (node instanceof DeleteDataNode) { + final DeleteDataNode tree = (DeleteDataNode) node; + final long first = Math.max(start, tree.getDeleteStartTime()); + final long last = Math.min(end, tree.getDeleteEndTime()); + return first > last + ? Optional.empty() + : Optional.of(new DeleteDataNode(tree.getPlanNodeId(), tree.getPathList(), first, last)); + } + if (node instanceof RelationalDeleteDataNode) { + final RelationalDeleteDataNode table = (RelationalDeleteDataNode) node; + final List entries = new ArrayList<>(); + for (final TableDeletionEntry entry : table.getModEntries()) { + final long first = Math.max(start, entry.getStartTime()); + final long last = Math.min(end, entry.getEndTime()); + if (first <= last) { + entries.add(new TableDeletionEntry(entry.getPredicate(), new TimeRange(first, last))); + } + } + return entries.isEmpty() + ? Optional.empty() + : Optional.of( + new RelationalDeleteDataNode( + table.getPlanNodeId(), entries, table.getDatabaseName())); + } + throw new IllegalArgumentException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97, + node.getClass().getName())); + } + + protected void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) + throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + final List requests = requestSupplier.get(); + if (!requests.isEmpty()) { + writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event)); + } + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + @FunctionalInterface + protected interface RequestSupplier { + List get() throws Exception; + } + + protected void handleUnsupported(final Event event) throws Exception { + if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) { + writer.recordSkippedEvent( + System.currentTimeMillis(), event == null ? "null" : event.getClass().getName()); + return; + } + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97, + event == null ? "null" : event.getClass().getName())); + } + + protected UUID eventId(final EnrichedEvent event) { + return UUID.nameUUIDFromBytes( + (event.getClass().getName() + + '\0' + + event.getPipeName() + + '\0' + + event.getCreationTime() + + '\0' + + event.getRegionId() + + '\0' + + commitIds(event)) + .getBytes(StandardCharsets.UTF_8)); + } + + private String commitIds(final EnrichedEvent event) { + try { + return String.valueOf(event.getCommitIds()); + } catch (final UnsupportedOperationException ignored) { + return Long.toString(event.getCommitId()); + } + } + + protected String metadata(final Event event) { + if (event instanceof EnrichedEvent) { + final EnrichedEvent enriched = (EnrichedEvent) event; + return "pipe=" + + enriched.getPipeName() + + ";creationTime=" + + enriched.getCreationTime() + + ";regionId=" + + enriched.getRegionId() + + ";commitIds=" + + commitIds(enriched) + + ";eventType=" + + event.getClass().getName(); + } + return "eventType=" + (event == null ? "null" : event.getClass().getName()); + } + + @Override + public void close() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 481ef55485d39..948b15584d780 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics; import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics; import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionSourceMetrics; @@ -137,6 +138,7 @@ public class IoTDBDataRegionSource extends IoTDBSource { private PipeHistoricalDataRegionSource historicalSource; private PipeRealtimeDataRegionSource realtimeSource; + private boolean logicalBackupCompletionSent; private DataRegionWatermarkInjector watermarkInjector; @@ -622,6 +624,15 @@ public Event supply() throws Exception { } if (Objects.isNull(event)) { event = realtimeSource.supply(); + if (event == null + && !logicalBackupCompletionSent + && realtimeSource.isLogicalBackupDrained()) { + final PipeTerminateEvent completion = + new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, regionId, true); + completion.increaseReferenceCount(IoTDBDataRegionSource.class.getName()); + logicalBackupCompletionSent = true; + event = completion; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index 801051b991c75..42ae3d1ef9ba2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -154,6 +154,8 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor { protected String userName; protected String cliHostname; protected boolean skipIfNoPrivileges = true; + private long logicalBackupUntil = -1; + private final Object logicalBackupFence = new Object(); protected PipeRealtimeDataRegionSource() { // Do nothing @@ -226,6 +228,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { public void customize( final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) throws Exception { + logicalBackupUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L); final PipeRuntimeEnvironment environment = configuration.getRuntimeEnvironment(); final Pair insertionDeletionListeningOptionPair = @@ -370,6 +373,28 @@ private void clearPendingQueue() { * @param event the {@link Event} from the {@link StorageEngine} */ public final void extract(final PipeRealtimeEvent event) { + if (logicalBackupUntil > 0) { + synchronized (logicalBackupFence) { + if (System.currentTimeMillis() >= logicalBackupUntil) { + event.decreaseReferenceCount(PipeRealtimeDataRegionSource.class.getName(), false); + return; + } + extractInternal(event); + } + } else { + extractInternal(event); + } + } + + public boolean isLogicalBackupDrained() { + synchronized (logicalBackupFence) { + return logicalBackupUntil > 0 + && System.currentTimeMillis() >= logicalBackupUntil + && pendingQueue.isEmpty(); + } + } + + private void extractInternal(final PipeRealtimeEvent event) { // The progress report event shall be directly extracted if (event.getEvent() instanceof ProgressReportEvent) { extractProgressReportEvent(event); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java index ccd72ffc77a70..98c9f201a2e2e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java @@ -38,6 +38,27 @@ public class PipeSinkSubtaskManagerTest { + @Test + public void testLogicalBackupUsesOneWriterPerRegion() { + for (final BuiltinPipePlugin plugin : + new BuiltinPipePlugin[] { + BuiltinPipePlugin.LOGICAL_BACKUP_SINK, BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR + }) { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_KEY, plugin.getPipePluginName()); + attributes.put(PipeSinkConstant.SINK_IOTDB_PARALLEL_TASKS_KEY, "3"); + attributes.put(PipeSinkConstant.SINK_SERIALIZE_BY_REGION_KEY, "false"); + final PipeParameters parameters = new PipeParameters(attributes); + Assert.assertEquals(1, PipeSinkSubtaskManager.calculateSinkSubtaskNum(parameters, -1)); + Assert.assertNotEquals( + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -1), + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -2)); + Assert.assertNotEquals( + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE), + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE - 1)); + } + } + @Test public void testSubtasksAreSharedOnlyWithinSamePipe() { // Initialize the task agent used by PipeEventCommitManager before registering subtasks. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java new file mode 100644 index 0000000000000..f2f0407d9b9e3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class LogicalBackupSinkTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testTemporaryValidationDoesNotCreateBackupStreams() throws Exception { + final Path directory = temporaryFolder.getRoot().toPath().resolve("backup"); + final PipeParameters parameters = + new PipeParameters( + Map.of( + "sink.dir", + directory.toString(), + "sink.backup-id", + "backup", + "sink.resume", + "append")); + try (final LogicalBackupSink sink = new LogicalBackupSink("data")) { + sink.validate(new PipeParameterValidator(parameters)); + sink.customize( + parameters, + new PipeTaskRuntimeConfiguration(new PipeTaskTemporaryRuntimeEnvironment("pipe"))); + sink.handshake(); + } + Assert.assertFalse(Files.exists(directory)); + } + + @Test + public void testDeleteTimeFilterPreservesPathsAndNeverExpandsTheRange() throws Exception { + final DeleteDataNode original = + new DeleteDataNode( + new PlanNodeId("deletion"), + Collections.singletonList(new MeasurementPath("root.backup.d.s1")), + 10L, + 20L); + final DeleteDataNode bounded = + (DeleteDataNode) LogicalBackupSink.boundedDeletion(original, 12L, 15L).get(); + Assert.assertEquals(original.getPathList(), bounded.getPathList()); + Assert.assertEquals(12L, bounded.getDeleteStartTime()); + Assert.assertEquals(15L, bounded.getDeleteEndTime()); + Assert.assertEquals(10L, original.getDeleteStartTime()); + Assert.assertEquals(20L, original.getDeleteEndTime()); + Assert.assertFalse(LogicalBackupSink.boundedDeletion(original, 21L, 30L).isPresent()); + } + + @Test + public void testInsertNodeIsFilteredBeforeItIsArchived() throws Exception { + final InsertTabletNode node = + new InsertTabletNode( + new PlanNodeId("backup-test"), + new PartialPath("root.backup.d"), + true, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT32}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32) + }, + new long[] {1, 2, 3}, + null, + new Object[] {new int[] {10, 20, 30}, new int[] {40, 50, 60}}, + 3); + final PipeInsertNodeTabletInsertionEvent event = + new PipeInsertNodeTabletInsertionEvent( + false, + "root.backup", + node, + null, + 0, + null, + new IoTDBTreePattern("root.backup.d.s1"), + null, + null, + null, + null, + true, + 2L, + 2L); + final List requests = new LogicalBackupSink("data").requests(event); + Assert.assertEquals(1, requests.size()); + final InsertTabletStatement restored = + PipeTransferTabletRawReqV2.fromTPipeTransferReq(requests.get(0)).constructStatement(); + Assert.assertArrayEquals(new String[] {"s1"}, restored.getMeasurements()); + Assert.assertArrayEquals(new long[] {2}, restored.getTimes()); + Assert.assertArrayEquals(new int[] {20}, (int[]) restored.getColumns()[0]); + Assert.assertTrue(restored.isAligned()); + } + + @Test + public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS); + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + + attributes.remove(PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + attributes.put( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + @Test + public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + new LogicalBackupSink("schema") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + @Test + public void testConfigRegionRequiresExplicitAppendResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + assertInvalid(new LogicalBackupSink("config"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("config") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + private static void assertInvalid( + final LogicalBackupSink sink, final Map attributes) throws Exception { + try { + sink.validate(new PipeParameterValidator(new PipeParameters(new HashMap<>(attributes)))); + Assert.fail(); + } catch (final PipeParameterNotValidException expected) { + Assert.assertTrue(expected.getMessage().contains("append")); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java index d935821cdba4e..0f02b31b91854 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java @@ -34,6 +34,26 @@ public class PipeRealtimeDataRegionSourceTest { private static final String TEST_REFERENCE_HOLDER = PipeRealtimeDataRegionSourceTest.class.getName(); + @Test + public void logicalBackupBoundaryRejectsNewEventsAndDrainsAcceptedEvents() throws Exception { + try (final ProgressReportTestSource source = new ProgressReportTestSource()) { + final java.lang.reflect.Field until = + PipeRealtimeDataRegionSource.class.getDeclaredField("logicalBackupUntil"); + until.setAccessible(true); + until.set(source, System.currentTimeMillis() + 60_000L); + final PipeRealtimeEvent accepted = createHeartbeatEvent(); + source.extract(accepted); + until.set(source, System.currentTimeMillis() - 1L); + final PipeRealtimeEvent late = createHeartbeatEvent(); + source.extract(late); + Assert.assertTrue(late.getEvent().isReleased()); + Assert.assertFalse(source.isLogicalBackupDrained()); + Assert.assertSame(accepted, source.supply()); + accepted.decreaseReferenceCount(TEST_REFERENCE_HOLDER, false); + Assert.assertTrue(source.isLogicalBackupDrained()); + } + } + @Test public void progressReportEventReleasesDroppedHeartbeatEvent() throws Exception { try (final ProgressReportTestSource source = new ProgressReportTestSource()) { diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3bb68a29562c3..a632dbff5ddef 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -139,6 +139,10 @@ com.google.guava guava + + com.google.code.gson + gson + com.google.code.findbugs jsr305 diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..deb5f8629fd7a --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "directory"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "manifest"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsyncPolicy"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "eventGroupId"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Logical backup directory already exists: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Logical backup event must contain at least one request"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Logical backup request body must not be null"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Logical backup record exceeds max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "Unsupported logical backup manifest format: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Logical backup manifest does not match %s: expected %s, found %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Logical backup directory is locked: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Logical backup writer is closed"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Incomplete logical backup segment header: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Invalid logical backup segment footer size: %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "Invalid logical backup record magic at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "Logical backup record header CRC mismatch at offset %d in %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "Invalid logical backup record length at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "Logical backup record payload CRC mismatch at offset %d in %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "Unknown logical backup record type %d in %s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Nested logical backup event groups in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Logical backup commit without begin in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "Logical backup operation index mismatch at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "Logical backup sequence mismatch at offset %d in %s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Incomplete logical backup record tail: %s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "Sealed logical backup segment contains an open event group: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Invalid logical backup segment magic: %s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "Unsupported logical backup segment major version %d: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Logical backup segment header CRC mismatch: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Invalid logical backup segment footer magic: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Logical backup segment footer ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Logical backup segment digest mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Logical backup segment footer CRC mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Logical backup segment footer metadata mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Logical backup event ID %s was already written with a different digest"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Invalid logical backup writer configuration: segment-size, max-record, fsync-batch and" + + " fsync-period must be positive"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "Logical backup request outside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "Logical backup control record inside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Logical backup segment ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Logical backup sequence gap in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Logical backup has no segments"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Invalid logical backup segment path: %s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "No logical backup manifest found under %s"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Duplicate logical backup stream ID: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Logical backup manifest has no segments: %s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "Unlisted logical backup segment %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Logical backup segment metadata mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Logical backup segment is not sealed: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Logical backup manifest counters do not match segment contents: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Logical backup sequence is not continuous across segments: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Logical backup manifest is invalid: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Logical backup contains skipped events: %s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "Logical backup request version %d is not supported"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "Logical backup request type %d is not allowed"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "Duplicate logical backup event group ID: %s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "Symbolic links are not allowed in logical backup directory paths: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "Logical backup directory is unavailable: %s"; + + private LogicalBackupMessages() {} + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 = + "Invalid logical backup root manifest, file, checksum or state: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB = + "Invalid logical backup writer settings."; + public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 = + "ConfigRegion and DataRegion backups require explicit sink.resume=append and a shared backup directory."; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB = + "Invalid logical backup setting: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 = + "Unsupported logical backup event: %s"; +} diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..b4cf5a28670c9 --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "目录"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "清单"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsync 策略"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "事件组 ID"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Pipe 逻辑备份目录已存在：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Pipe 逻辑备份事件必须至少包含一个请求"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Pipe 逻辑备份请求体不能为空"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Pipe 逻辑备份记录超过 max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "不支持的 Pipe 逻辑备份清单格式：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Pipe 逻辑备份清单的 %s 不匹配：预期 %s，实际 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Pipe 逻辑备份目录已被锁定：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Pipe 逻辑备份写入器已关闭"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Pipe 逻辑备份 segment 文件头不完整：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Pipe 逻辑备份 segment 文件尾大小无效：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "偏移 %d 处的 Pipe 逻辑备份记录魔数无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "偏移 %d 处的 Pipe 逻辑备份记录头 CRC 不匹配，文件：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "偏移 %d 处的 Pipe 逻辑备份记录长度无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "偏移 %d 处的 Pipe 逻辑备份记录载荷 CRC 不匹配，文件：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "未知的 Pipe 逻辑备份记录类型 %d，文件：%s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Pipe 逻辑备份事件组发生嵌套，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Pipe 逻辑备份事件组缺少开始记录，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "偏移 %d 处的 Pipe 逻辑备份操作序号不匹配，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "偏移 %d 处的 Pipe 逻辑备份 sequence 不连续，文件：%s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Pipe 逻辑备份记录尾部不完整：%s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "已封存的 Pipe 逻辑备份 segment 包含未提交事件组：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Pipe 逻辑备份 segment 魔数无效：%s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "不支持的 Pipe 逻辑备份 segment 主版本 %d：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Pipe 逻辑备份 segment 文件头 CRC 不匹配：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Pipe 逻辑备份 segment 文件尾魔数无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Pipe 逻辑备份 segment 文件尾 ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Pipe 逻辑备份 segment 摘要不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Pipe 逻辑备份 segment 文件尾 CRC 不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Pipe 逻辑备份 segment 文件尾元数据不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Pipe 逻辑备份事件 ID %s 已使用不同摘要写入"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Pipe 逻辑备份写入器配置无效：segment-size、max-record、fsync-batch 和 fsync-period" + + " 必须为正数"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "偏移 %d 处的 Pipe 逻辑备份请求不在事件组中，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "偏移 %d 处的 Pipe 逻辑备份控制记录位于事件组内，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Pipe 逻辑备份 segment ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Pipe 逻辑备份 sequence 在 %s 中不连续"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Pipe 逻辑备份不包含 segment"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Pipe 逻辑备份 segment 路径无效：%s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "在 %s 下未找到 Pipe 逻辑备份 manifest"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Pipe 逻辑备份 stream ID 重复：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Pipe 逻辑备份 manifest 不包含 segment：%s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "未在 manifest 中列出 Pipe 逻辑备份 segment：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Pipe 逻辑备份 segment 元数据不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Pipe 逻辑备份 segment 尚未封存：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Pipe 逻辑备份 manifest 计数与 segment 内容不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Pipe 逻辑备份 sequence 在 segment 之间不连续：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Pipe 逻辑备份 manifest 无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Pipe 逻辑备份包含已跳过的事件：%s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "不支持逻辑备份请求版本 %d"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "不允许逻辑备份请求类型 %d"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "逻辑备份事件组 ID 重复：%s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "逻辑备份目录路径中不允许使用符号链接：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "逻辑备份目录不可用：%s"; + + private LogicalBackupMessages() {} + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 = + "逻辑备份根清单、文件、校验和或状态无效：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB = + "逻辑备份写入参数无效。"; + public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 = + "ConfigRegion 和 DataRegion 备份必须显式设置 sink.resume=append，并使用共享备份目录。"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB = + "逻辑备份参数无效：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 = + "不支持的逻辑备份事件：%s"; +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 96ab486ebd7b8..e6379092d7166 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.writeback.WriteBackSink; @@ -93,6 +94,7 @@ public enum BuiltinPipePlugin { WEBSOCKET_CONNECTOR("websocket-connector", WebSocketSink.class), OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class), WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class), + LOGICAL_BACKUP_CONNECTOR("logical-backup-connector", LogicalBackupSink.class), DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class), IOTDB_THRIFT_SINK("iotdb-thrift-sink", IoTDBThriftSink.class, true), @@ -104,6 +106,7 @@ public enum BuiltinPipePlugin { WEBSOCKET_SINK("websocket-sink", WebSocketSink.class), OPC_DA_SINK("opc-da-sink", OpcDaSink.class), WRITE_BACK_SINK("write-back-sink", WriteBackSink.class), + LOGICAL_BACKUP_SINK("logical-backup-sink", LogicalBackupSink.class), SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class), IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class), // Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename. @@ -225,6 +228,7 @@ public String getClassName() { WEBSOCKET_CONNECTOR.getPipePluginName().toUpperCase(), OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(), WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(), + LOGICAL_BACKUP_CONNECTOR.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), // Sinks @@ -233,6 +237,7 @@ public String getClassName() { IOTDB_LEGACY_PIPE_SINK.getPipePluginName().toUpperCase(), WEBSOCKET_SINK.getPipePluginName().toUpperCase(), OPC_DA_SINK.getPipePluginName().toUpperCase(), + LOGICAL_BACKUP_SINK.getPipePluginName().toUpperCase(), SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase()))); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..d5fb9b22021ef --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.PlaceholderSink; + +public class LogicalBackupSink extends PlaceholderSink {} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java index f95ed6ce29841..438714cfa4f57 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java @@ -83,6 +83,55 @@ public class PipeSinkConstant { public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first"; public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true; + public static final String CONNECTOR_LOGICAL_BACKUP_DIR_KEY = "connector.dir"; + public static final String SINK_LOGICAL_BACKUP_DIR_KEY = "sink.dir"; + public static final String CONNECTOR_LOGICAL_BACKUP_ID_KEY = "connector.backup-id"; + public static final String SINK_LOGICAL_BACKUP_ID_KEY = "sink.backup-id"; + public static final String CONNECTOR_LOGICAL_BACKUP_RESUME_KEY = "connector.resume"; + public static final String SINK_LOGICAL_BACKUP_RESUME_KEY = "sink.resume"; + public static final String LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS = "fail-if-exists"; + public static final String LOGICAL_BACKUP_RESUME_APPEND = "append"; + public static final String LOGICAL_BACKUP_RESUME_NEW = "new"; + public static final String LOGICAL_BACKUP_RESUME_DEFAULT_VALUE = + LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS; + public static final String CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = + "connector.segment-size-bytes"; + public static final String SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = "sink.segment-size-bytes"; + public static final long LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE = 256L * MB; + public static final String CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = + "connector.max-record-bytes"; + public static final String SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = "sink.max-record-bytes"; + public static final int LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE = 64 * 1024 * 1024; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "connector.fsync-policy"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "sink.fsync-policy"; + public static final String LOGICAL_BACKUP_FSYNC_ALWAYS = "always"; + public static final String LOGICAL_BACKUP_FSYNC_BATCH = "batch"; + public static final String LOGICAL_BACKUP_FSYNC_PERIODIC = "periodic"; + public static final String LOGICAL_BACKUP_FSYNC_NONE = "none"; + public static final String LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE = + LOGICAL_BACKUP_FSYNC_ALWAYS; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "connector.fsync-batch-operations"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "sink.fsync-batch-operations"; + public static final int LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = + "connector.fsync-period-ms"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = "sink.fsync-period-ms"; + public static final long LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "connector.on-unsupported-event"; + public static final String SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "sink.on-unsupported-event"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL = "fail"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP = "skip"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE = + LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL; + public static final String CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = + "connector.include-heartbeat"; + public static final String SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = "sink.include-heartbeat"; + public static final boolean LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE = false; + public static final String CONNECTOR_SERIALIZE_BY_REGION_KEY = "connector.serialize-by-region"; public static final String SINK_SERIALIZE_BY_REGION_KEY = "sink.serialize-by-region"; public static final boolean CONNECTOR_SERIALIZE_BY_REGION_DEFAULT_VALUE = false; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java new file mode 100644 index 0000000000000..f90522acbf820 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java @@ -0,0 +1,620 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; + +import com.google.gson.Gson; +import com.google.gson.JsonParseException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LogicalBackupArchiveReader { + + private static final Gson GSON = new Gson(); + private static final Set REPLAYABLE_REQUEST_TYPES = + Set.of( + PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType(), + PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), + PipeRequestType.TRANSFER_PLAN_NODE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), + PipeRequestType.TRANSFER_CONFIG_PLAN.getType(), + PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType(), + PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType()); + + public List read(final Path source, final boolean allowIncomplete) + throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + LogicalBackupCatalog.rejectSymbolicLinks(normalizedSource); + final Path root = + Files.isDirectory(normalizedSource) ? normalizedSource : normalizedSource.getParent(); + final LogicalBackupCatalog catalog = + LogicalBackupCatalog.load( + Files.isDirectory(normalizedSource) + ? normalizedSource.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME) + : normalizedSource); + final List manifests = + catalog == null + ? discoverManifests(normalizedSource) + : catalogManifests(root, catalog, allowIncomplete); + if (manifests.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E, + source)); + } + final List streams = new ArrayList<>(); + final Set streamIds = new HashSet<>(); + LogicalBackupManifest archiveIdentity = null; + for (final Path manifest : manifests) { + final BackupStream stream = readStream(manifest, allowIncomplete); + stream.catalog = catalog; + if (archiveIdentity == null) { + archiveIdentity = stream.getManifest(); + } else { + validateArchiveIdentity(archiveIdentity, stream.getManifest()); + } + if (!streamIds.add(stream.getManifest().streamId)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE, + stream.getManifest().streamId)); + } + streams.add(stream); + } + if (catalog != null) { + validateCatalog(root, catalog, streams); + } + streams.sort( + Comparator.comparingInt( + (BackupStream stream) -> streamOrder(stream.getManifest().streamType)) + .thenComparing((BackupStream stream) -> stream.getManifest().streamId)); + return Collections.unmodifiableList(streams); + } + + private static List catalogManifests( + final Path root, final LogicalBackupCatalog catalog, final boolean allowIncomplete) + throws IOException { + if ((!"SEALED".equals(catalog.status) && !"INCOMPLETE".equals(catalog.status)) + || (!allowIncomplete + && ("INCOMPLETE".equals(catalog.status) || catalog.skippedEventCount > 0))) { + throw LogicalBackupCatalog.invalid(root); + } + final List manifests = new ArrayList<>(); + final Set files = new HashSet<>(); + files.add(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + if (entry == null || entry.manifest == null || entry.segments == null) { + throw LogicalBackupCatalog.invalid(root); + } + final Path manifest = validateCatalogFile(root, entry.manifest, files); + if (!LogicalBackupFormat.MANIFEST_FILE_NAME.equals(manifest.getFileName().toString())) { + throw LogicalBackupCatalog.invalid(manifest); + } + manifests.add(manifest); + for (final LogicalBackupCatalog.FileEntry segment : entry.segments) { + validateCatalogFile(root, segment, files); + } + } + try (final Stream paths = Files.walk(root)) { + for (final Path path : paths.toList()) { + if (Files.isSymbolicLink(path) + || (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS) + && !files.contains(path) + && !".backup.lock".equals(path.getFileName().toString()))) { + throw LogicalBackupCatalog.invalid(path); + } + } + } + return manifests; + } + + private static Path validateCatalogFile( + final Path root, final LogicalBackupCatalog.FileEntry entry, final Set files) + throws IOException { + if (entry == null) { + throw LogicalBackupCatalog.invalid(root); + } + final Path path = LogicalBackupCatalog.resolve(root, entry.file); + if (!files.add(path) + || !Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS) + || entry.sizeBytes != Files.size(path) + || !LogicalBackupCatalog.sha256(path).equals(entry.sha256)) { + throw LogicalBackupCatalog.invalid(path); + } + return path; + } + + private static void validateCatalog( + final Path root, final LogicalBackupCatalog catalog, final List streams) + throws IOException { + long skipped = 0; + final Map counts = new LinkedHashMap<>(); + final Map highWatermarks = new LinkedHashMap<>(); + for (int index = 0; index < streams.size(); index++) { + final BackupStream stream = streams.get(index); + final LogicalBackupManifest manifest = stream.manifest; + final LogicalBackupCatalog.StreamEntry entry = catalog.streams.get(index); + if (!manifest.streamId.equals(entry.streamId) + || !manifest.streamType.equals(entry.streamType) + || !manifest.backupId.equals(catalog.backupId) + || !java.util.Objects.equals(manifest.pipeName, catalog.pipeName) + || !java.util.Objects.equals(manifest.sourceClusterId, catalog.sourceClusterId) + || !java.util.Objects.equals(manifest.sourceVersion, catalog.sourceVersion) + || !java.util.Objects.equals(manifest.timestampPrecision, catalog.timestampPrecision) + || !java.util.Objects.equals(manifest.captureOptions, catalog.captureOptions) + || entry.segments.size() != manifest.segments.size()) { + throw LogicalBackupCatalog.invalid(root); + } + if (("data".equals(manifest.streamType) && !catalog.scope.contains("data")) + || ("schema".equals(manifest.streamType) && !catalog.scope.contains("schema")) + || ("config".equals(manifest.streamType) + && !catalog.scope.contains("config") + && !catalog.scope.contains("schema"))) { + throw LogicalBackupCatalog.invalid(root); + } + for (int segment = 0; segment < manifest.segments.size(); segment++) { + if (!LogicalBackupCatalog.resolve(root, entry.segments.get(segment).file) + .equals(stream.manifestPath.getParent().resolve(manifest.segments.get(segment).file))) { + throw LogicalBackupCatalog.invalid(root); + } + } + skipped += manifest.skippedEventCount; + manifest.operationCounts.forEach((key, value) -> counts.merge(key, value, Long::sum)); + highWatermarks.put(manifest.streamId, manifest.lastDurableSequence); + } + if (skipped != catalog.skippedEventCount + || !counts.equals(catalog.operationCounts) + || !highWatermarks.equals(catalog.sourceHighWatermark)) { + throw LogicalBackupCatalog.invalid(root); + } + } + + private static int streamOrder(final String streamType) { + if ("config".equalsIgnoreCase(streamType)) { + return 0; + } + if ("schema".equalsIgnoreCase(streamType)) { + return 1; + } + return 2; + } + + private static void validateArchiveIdentity( + final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException { + validateArchiveIdentityField("backupId", expected.backupId, actual.backupId); + validateArchiveIdentityField("pipeName", expected.pipeName, actual.pipeName); + validateArchiveIdentityField( + "pipeCreationTime", expected.pipeCreationTime, actual.pipeCreationTime); + validateArchiveIdentityField( + "sourceClusterId", expected.sourceClusterId, actual.sourceClusterId); + validateArchiveIdentityField("sourceVersion", expected.sourceVersion, actual.sourceVersion); + validateArchiveIdentityField( + "timestampPrecision", expected.timestampPrecision, actual.timestampPrecision); + } + + private static void validateArchiveIdentityField( + final String field, final Object expected, final Object actual) throws IOException { + if (!java.util.Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete) + throws IOException { + final LogicalBackupManifest manifest; + LogicalBackupCatalog.rejectSymbolicLinks(manifestPath); + if (Files.size(manifestPath) > 16 * 1024 * 1024) { + throw LogicalBackupCatalog.invalid(manifestPath); + } + try { + manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + validateManifest(manifest, manifestPath); + if ((!"SEALED".equals(manifest.status) && !"INCOMPLETE".equals(manifest.status)) + || (!allowIncomplete + && (manifest.skippedEventCount > 0 || "INCOMPLETE".equals(manifest.status)))) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + + final Path streamDirectory = manifestPath.getParent().toAbsolutePath().normalize(); + final LogicalBackupSegmentReader segmentReader = + new LogicalBackupSegmentReader(manifest.maxRecordBytes); + final List records = new ArrayList<>(); + final Map operationCounts = new LinkedHashMap<>(); + final Set listedSegments = new HashSet<>(); + long expectedSequence = -1; + long expectedSegmentId = 0; + boolean allSealed = true; + + for (int index = 0; index < manifest.segments.size(); index++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(index); + if (segment == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + "null")); + } + final Path segmentPath = resolveSegmentPath(streamDirectory, segment.file); + if (segmentPath == null + || Files.isSymbolicLink(segmentPath) + || !Files.isRegularFile(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + segment.file)); + } + if (!listedSegments.add(segmentPath)) { + throw LogicalBackupCatalog.invalid(segmentPath); + } + final LogicalBackupSegmentReader.ScanResult scan = segmentReader.scan(segmentPath, false); + if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if (!scan.isSealed() || scan.hasOpenEventGroup()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF, + segmentPath)); + } + allSealed &= scan.isSealed(); + if (scan.isSealed()) { + validateSegmentManifest(segment, scan, segmentPath); + } + for (final LogicalBackupRecord record : scan.getRecords()) { + validateReplayableRequest(record); + } + + final List committedRecords = + scan.hasOpenEventGroup() ? removeOpenEventGroup(scan.getRecords()) : scan.getRecords(); + for (final LogicalBackupRecord record : committedRecords) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E, + manifestPath)); + } + expectedSequence = record.getSequence() + 1; + records.add(record); + operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + } + } + + validateUnlistedSegments(streamDirectory, listedSegments); + if (allSealed) { + final long skippedEventCount = + operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L); + if (!allowIncomplete && skippedEventCount > 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + if (skippedEventCount != manifest.skippedEventCount + || !operationCounts.equals(manifest.operationCounts) + || (records.isEmpty() + ? manifest.firstSequence != -1 || manifest.lastSequence != -1 + : records.get(0).getSequence() != manifest.firstSequence + || records.get(records.size() - 1).getSequence() != manifest.lastSequence)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB, + manifestPath)); + } + } + return new BackupStream(manifestPath, manifest, records, toEventGroups(records)); + } + + private static void validateReplayableRequest(final LogicalBackupRecord record) + throws IOException { + if (record.getRecordType() != LogicalBackupRecordType.PIPE_REQUEST) { + return; + } + if (record.getRequestVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion()) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7, + record.getRequestVersion())); + } + if (!REPLAYABLE_REQUEST_TYPES.contains(record.getRequestType())) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38, + record.getRequestType())); + } + } + + private static List discoverManifests(final Path source) throws IOException { + if (!Files.isSymbolicLink(source) + && Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + && LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())) { + return Collections.singletonList(source); + } + if (!Files.isDirectory(source)) { + return Collections.emptyList(); + } + try (final Stream paths = Files.walk(source)) { + return paths + .filter(path -> !Files.isSymbolicLink(path)) + .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + .filter( + path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString())) + .sorted() + .collect(Collectors.toList()); + } + } + + private static Path resolveSegmentPath(final Path streamDirectory, final String segmentFile) { + if (segmentFile == null) { + return null; + } + final Path relativePath; + try { + relativePath = Path.of(segmentFile); + } catch (final RuntimeException e) { + return null; + } + if (relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + return null; + } + final Path segmentPath = streamDirectory.resolve(relativePath).normalize(); + return segmentPath.startsWith(streamDirectory) + && segmentPath.getParent().equals(streamDirectory) + ? segmentPath + : null; + } + + private static void validateManifest( + final LogicalBackupManifest manifest, final Path manifestPath) throws IOException { + if (manifest == null + || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion) + || manifest.streamId == null + || !List.of("config", "schema", "data").contains(manifest.streamType) + || manifest.backupId == null + || manifest.timestampPrecision == null + || manifest.maxRecordBytes <= 0 + || manifest.maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || manifest.skippedEventCount < 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (manifest.segments == null || manifest.segments.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138, + manifestPath)); + } + if (manifest.operationCounts == null) { + manifest.operationCounts = new LinkedHashMap<>(); + } + } + + private static void validateSegmentManifest( + final LogicalBackupManifest.Segment segment, + final LogicalBackupSegmentReader.ScanResult scan, + final Path segmentPath) + throws IOException { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (!"SEALED".equals(segment.status) + || segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + private static void validateUnlistedSegments( + final Path streamDirectory, final Set listedSegments) throws IOException { + try (final Stream paths = Files.list(streamDirectory)) { + final Path unlisted = + paths + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .map(path -> path.toAbsolutePath().normalize()) + .filter(path -> !listedSegments.contains(path)) + .findFirst() + .orElse(null); + if (unlisted != null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + unlisted)); + } + } + } + + private static List removeOpenEventGroup( + final List records) { + for (int i = records.size() - 1; i >= 0; i--) { + if (records.get(i).getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + return records.subList(0, i); + } + } + return records; + } + + private static List toEventGroups(final List records) + throws IOException { + final List groups = new ArrayList<>(); + final Set eventGroupIds = new HashSet<>(); + UUID eventGroupId = null; + long firstSequence = -1; + String metadata = ""; + final List requests = new ArrayList<>(); + for (final LogicalBackupRecord record : records) { + if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventGroupId = record.getEventGroupId(); + if (!eventGroupIds.add(eventGroupId)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89, + eventGroupId)); + } + firstSequence = record.getSequence(); + metadata = record.getMetadata(); + requests.clear(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + requests.add(record); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + groups.add( + new EventGroup( + eventGroupId, + firstSequence, + record.getSequence(), + metadata, + new ArrayList<>(requests))); + eventGroupId = null; + requests.clear(); + } + } + return Collections.unmodifiableList(groups); + } + + public static class BackupStream { + private final Path manifestPath; + private final LogicalBackupManifest manifest; + private final List records; + private final List eventGroups; + private LogicalBackupCatalog catalog; + + private BackupStream( + final Path manifestPath, + final LogicalBackupManifest manifest, + final List records, + final List eventGroups) { + this.manifestPath = manifestPath; + this.manifest = manifest; + this.records = Collections.unmodifiableList(records); + this.eventGroups = eventGroups; + } + + public Path getManifestPath() { + return manifestPath; + } + + public LogicalBackupManifest getManifest() { + return manifest; + } + + public LogicalBackupCatalog getCatalog() { + return catalog; + } + + public List getRecords() { + return records; + } + + public List getEventGroups() { + return eventGroups; + } + } + + public static class EventGroup { + private final UUID eventGroupId; + private final long firstSequence; + private final long lastSequence; + private final String metadata; + private final List requests; + + private EventGroup( + final UUID eventGroupId, + final long firstSequence, + final long lastSequence, + final String metadata, + final List requests) { + this.eventGroupId = eventGroupId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.metadata = metadata; + this.requests = Collections.unmodifiableList(requests); + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public String getMetadata() { + return metadata; + } + + public List getRequests() { + return requests; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java new file mode 100644 index 0000000000000..3b9d79b1fed12 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** The portable, authoritative root manifest of a backup task. */ +public class LogicalBackupCatalog { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + public String formatName = LogicalBackupFormat.FORMAT_NAME; + public String formatVersion = LogicalBackupFormat.FORMAT_VERSION; + public String backupId; + public String pipeName; + public String status = "WRITING"; + public String sourceClusterId; + public String sourceVersion; + public String timestampPrecision; + public List scope = new ArrayList<>(); + public List model = new ArrayList<>(); + public Map filters = new LinkedHashMap<>(); + public Map captureOptions = new LinkedHashMap<>(); + public boolean includeAuth; + public String mode = "snapshot"; + public String createdAt = Instant.now().toString(); + public String capturedAt; + public String sealedAt; + public Map sourceHighWatermark = new LinkedHashMap<>(); + public Map operationCounts = new LinkedHashMap<>(); + public long skippedEventCount; + public boolean recovered; + public List streams = new ArrayList<>(); + + public static class StreamEntry { + public String streamId; + public String streamType; + public FileEntry manifest; + public List segments = new ArrayList<>(); + } + + public static class FileEntry { + public String file; + public long sizeBytes; + public String sha256; + } + + public void seal(final Path root, final List backupStreams) throws IOException { + collect(root, backupStreams); + write(root); + } + + public void collect(final Path root, final List backupStreams) throws IOException { + streams.clear(); + operationCounts.clear(); + sourceHighWatermark.clear(); + skippedEventCount = 0; + recovered = false; + for (final BackupStream stream : backupStreams) { + final LogicalBackupManifest manifest = stream.getManifest(); + if (!backupId.equals(manifest.backupId)) { + throw invalid(root); + } + if (streams.isEmpty()) { + sourceClusterId = manifest.sourceClusterId; + sourceVersion = manifest.sourceVersion; + timestampPrecision = manifest.timestampPrecision; + } + if (!java.util.Objects.equals(pipeName, manifest.pipeName) + || !java.util.Objects.equals(sourceClusterId, manifest.sourceClusterId) + || !java.util.Objects.equals(sourceVersion, manifest.sourceVersion) + || !java.util.Objects.equals(timestampPrecision, manifest.timestampPrecision) + || !captureOptions.equals(manifest.captureOptions)) { + throw invalid(root); + } + final StreamEntry entry = new StreamEntry(); + entry.streamId = manifest.streamId; + entry.streamType = manifest.streamType; + entry.manifest = describe(root, stream.getManifestPath()); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + entry.segments.add( + describe(root, stream.getManifestPath().getParent().resolve(segment.file))); + } + streams.add(entry); + manifest.operationCounts.forEach( + (key, value) -> operationCounts.merge(key, value, Long::sum)); + sourceHighWatermark.put(manifest.streamId, manifest.lastDurableSequence); + skippedEventCount += manifest.skippedEventCount; + recovered |= manifest.recovered; + } + capturedAt = Instant.now().toString(); + sealedAt = capturedAt; + status = skippedEventCount > 0 ? "INCOMPLETE" : "SEALED"; + } + + public void write(final Path root) throws IOException { + rejectSymbolicLinks(root); + Files.createDirectories(root); + final Path target = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (Files.isSymbolicLink(target)) { + throw invalid(target); + } + final Path temporary = Files.createTempFile(root, ".manifest-", ".tmp"); + try { + final ByteBuffer bytes = StandardCharsets.UTF_8.encode(GSON.toJson(this)); + try (final FileChannel channel = FileChannel.open(temporary, StandardOpenOption.WRITE)) { + while (bytes.hasRemaining()) { + channel.write(bytes); + } + channel.force(true); + } + try { + Files.move( + temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + /** Returns null for a legacy stream manifest; malformed root manifests always fail. */ + public static LogicalBackupCatalog load(final Path manifest) throws IOException { + rejectSymbolicLinks(manifest); + if (!Files.isRegularFile(manifest, LinkOption.NOFOLLOW_LINKS)) { + return null; + } + if (Files.size(manifest) > 16 * 1024 * 1024) { + throw invalid(manifest); + } + try { + final com.google.gson.JsonObject json = + GSON.fromJson(Files.readString(manifest), com.google.gson.JsonObject.class); + if (json == null) { + throw invalid(manifest); + } + if (json.has("streamId")) { + return null; + } + final LogicalBackupCatalog catalog = GSON.fromJson(json, LogicalBackupCatalog.class); + if (!LogicalBackupFormat.FORMAT_NAME.equals(catalog.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(catalog.formatVersion) + || catalog.backupId == null + || catalog.scope == null + || catalog.model == null + || catalog.filters == null + || catalog.captureOptions == null + || catalog.streams == null + || catalog.operationCounts == null + || catalog.sourceHighWatermark == null + || catalog.skippedEventCount < 0 + || catalog.scope.isEmpty() + || !List.of("config", "schema", "data").containsAll(catalog.scope) + || catalog.model.isEmpty() + || !List.of("tree", "table").containsAll(catalog.model) + || !List.of("WRITING", "RECOVERED", "SEALED", "INCOMPLETE").contains(catalog.status)) { + throw invalid(manifest); + } + return catalog; + } catch (final JsonParseException | IllegalStateException e) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3, + manifest), + e); + } + } + + public static FileEntry describe(final Path root, final Path path) throws IOException { + final Path normalizedRoot = root.toAbsolutePath().normalize(); + final Path normalizedPath = path.toAbsolutePath().normalize(); + if (!normalizedPath.startsWith(normalizedRoot)) { + throw invalid(path); + } + final FileEntry entry = new FileEntry(); + entry.file = normalizedRoot.relativize(normalizedPath).toString().replace('\\', '/'); + resolve(normalizedRoot, entry.file); + entry.sizeBytes = Files.size(normalizedPath); + entry.sha256 = sha256(normalizedPath); + return entry; + } + + public static Path resolve(final Path root, final String relative) throws IOException { + if (relative == null + || relative.isBlank() + || relative.indexOf('\\') >= 0 + || relative.indexOf(':') >= 0 + || relative.startsWith("/") + || relative.chars().anyMatch(Character::isISOControl)) { + throw invalid(root); + } + for (final String part : relative.split("/", -1)) { + if (part.isEmpty() || ".".equals(part) || "..".equals(part)) { + throw invalid(root); + } + } + final Path normalizedRoot = root.toAbsolutePath().normalize(); + final Path path = normalizedRoot.resolve(relative).normalize(); + if (!path.startsWith(normalizedRoot) || path.equals(normalizedRoot)) { + throw invalid(path); + } + rejectSymbolicLinks(path); + return path; + } + + public static void rejectSymbolicLinks(final Path path) throws IOException { + final Path absolute = path.toAbsolutePath().normalize(); + Path current = absolute.getRoot(); + for (final Path part : absolute) { + current = current.resolve(part); + if (Files.isSymbolicLink(current)) { + throw invalid(current); + } + } + } + + public static String sha256(final Path path) throws IOException { + final MessageDigest digest = LogicalBackupFormat.newSha256(); + try (final InputStream input = Files.newInputStream(path)) { + final byte[] buffer = new byte[64 * 1024]; + int length; + while ((length = input.read(buffer)) != -1) { + digest.update(buffer, 0, length); + } + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + static IOException invalid(final Path path) { + return new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3, + path)); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java new file mode 100644 index 0000000000000..6309595202b52 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.zip.CRC32C; + +public final class LogicalBackupFormat { + + public static final String FORMAT_NAME = "iotdb-pipe-logical-backup"; + public static final String FORMAT_VERSION = "1.0"; + public static final short MAJOR_VERSION = 1; + public static final short MINOR_VERSION = 0; + public static final long SEGMENT_MAGIC = 0x494F545057414C31L; + public static final int RECORD_MAGIC = 0x50575231; + public static final int FOOTER_MAGIC = 0x50575346; + public static final int SEGMENT_HEADER_SIZE = 32; + public static final int RECORD_HEADER_SIZE = 56; + public static final int SEGMENT_FOOTER_SIZE = 96; + public static final int SHA256_SIZE = 32; + public static final int MAX_RECORD_BYTES = 256 * 1024 * 1024; + public static final String MANIFEST_FILE_NAME = "manifest.json"; + + private LogicalBackupFormat() {} + + public static int crc32c(final byte[] bytes, final int offset, final int length) { + final CRC32C crc32c = new CRC32C(); + crc32c.update(bytes, offset, length); + return (int) crc32c.getValue(); + } + + public static MessageDigest newSha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + public static String toHex(final byte[] bytes) { + return HexFormat.of().formatHex(bytes); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java new file mode 100644 index 0000000000000..891ed03ff6e81 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class LogicalBackupManifest { + + public String formatName = LogicalBackupFormat.FORMAT_NAME; + public String formatVersion = LogicalBackupFormat.FORMAT_VERSION; + public String backupId; + public String status; + public String pipeName; + public long pipeCreationTime; + public String sourceClusterId; + public String sourceVersion; + public String timestampPrecision; + public String createdAt; + public String closedAt; + public String sealedAt; + public Map captureOptions = new LinkedHashMap<>(); + public String streamId; + public int regionId; + public String sinkTaskId; + public String streamType; + public String fsyncPolicy; + public int fsyncBatchOperations; + public long fsyncPeriodMs; + public long segmentSizeBytes; + public int maxRecordBytes; + public long firstSequence = -1; + public long lastSequence = -1; + public long lastDurableSequence = -1; + public String lastEventGroupId; + public String lastEventDigest; + public long lastEventFirstSequence = -1; + public long skippedEventCount; + public boolean recovered; + public Map operationCounts = new LinkedHashMap<>(); + public List segments = new ArrayList<>(); + + public static class Segment { + public String file; + public long segmentId; + public long firstSequence = -1; + public long lastSequence = -1; + public long recordCount; + public long sizeBytes; + public String sha256; + public String status; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java new file mode 100644 index 0000000000000..b779c4fc4aa3e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.UUID; + +public class LogicalBackupRecord { + + private final LogicalBackupRecordType recordType; + private final long sequence; + private final UUID eventGroupId; + private final int operationIndex; + private final long eventTime; + private final byte requestVersion; + private final short requestType; + private final String metadata; + private final byte[] payload; + + public LogicalBackupRecord( + final LogicalBackupRecordType recordType, + final long sequence, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final byte requestVersion, + final short requestType, + final String metadata, + final byte[] payload) { + this.recordType = recordType; + this.sequence = sequence; + this.eventGroupId = eventGroupId; + this.operationIndex = operationIndex; + this.eventTime = eventTime; + this.requestVersion = requestVersion; + this.requestType = requestType; + this.metadata = metadata; + this.payload = payload; + } + + public LogicalBackupRecordType getRecordType() { + return recordType; + } + + public long getSequence() { + return sequence; + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public int getOperationIndex() { + return operationIndex; + } + + public long getEventTime() { + return eventTime; + } + + public byte getRequestVersion() { + return requestVersion; + } + + public short getRequestType() { + return requestType; + } + + public String getMetadata() { + return metadata; + } + + public byte[] getPayload() { + return payload; + } + + public TPipeTransferReq toTPipeTransferReq() { + return new TPipeTransferReq() + .setVersion(requestVersion) + .setType(requestType) + .setBody(ByteBuffer.wrap(payload)); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LogicalBackupRecord)) { + return false; + } + final LogicalBackupRecord that = (LogicalBackupRecord) obj; + return sequence == that.sequence + && operationIndex == that.operationIndex + && eventTime == that.eventTime + && requestVersion == that.requestVersion + && requestType == that.requestType + && recordType == that.recordType + && eventGroupId.equals(that.eventGroupId) + && metadata.equals(that.metadata) + && Arrays.equals(payload, that.payload); + } + + @Override + public int hashCode() { + int result = eventGroupId.hashCode(); + result = 31 * result + recordType.hashCode(); + result = 31 * result + Long.hashCode(sequence); + result = 31 * result + operationIndex; + result = 31 * result + Long.hashCode(eventTime); + result = 31 * result + requestVersion; + result = 31 * result + requestType; + result = 31 * result + metadata.hashCode(); + return 31 * result + Arrays.hashCode(payload); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java new file mode 100644 index 0000000000000..e188669548ca6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.HashMap; +import java.util.Map; + +public enum LogicalBackupRecordType { + EVENT_BEGIN((byte) 1), + PIPE_REQUEST((byte) 2), + EVENT_COMMIT((byte) 3), + HEARTBEAT((byte) 4), + SKIPPED_EVENT((byte) 5), + STREAM_END((byte) 6); + + private static final Map TYPE_MAP = new HashMap<>(); + + static { + for (final LogicalBackupRecordType type : values()) { + TYPE_MAP.put(type.code, type); + } + } + + private final byte code; + + LogicalBackupRecordType(final byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + public static LogicalBackupRecordType valueOf(final byte code) { + return TYPE_MAP.get(code); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java new file mode 100644 index 0000000000000..e400d6133c27d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java @@ -0,0 +1,541 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; + +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupSegmentReader { + + private final int maxRecordBytes; + + public LogicalBackupSegmentReader(final int maxRecordBytes) { + this.maxRecordBytes = maxRecordBytes; + } + + public ScanResult scan(final Path segment, final boolean allowIncompleteTail) throws IOException { + try (final RandomAccessFile reader = new RandomAccessFile(segment.toFile(), "r")) { + final long fileLength = reader.length(); + if (fileLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new EOFException( + String.format( + LogicalBackupMessages + .EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A, + segment)); + } + + final byte[] segmentHeader = readBytes(reader, LogicalBackupFormat.SEGMENT_HEADER_SIZE); + validateSegmentHeader(segmentHeader, segment); + final ByteBuffer segmentHeaderBuffer = wrap(segmentHeader); + segmentHeaderBuffer.position(Long.BYTES + Short.BYTES * 2); + final long segmentId = segmentHeaderBuffer.getLong(); + final long createdAt = segmentHeaderBuffer.getLong(); + + final MessageDigest digest = LogicalBackupFormat.newSha256(); + digest.update(segmentHeader); + final List records = new ArrayList<>(); + long validLength = LogicalBackupFormat.SEGMENT_HEADER_SIZE; + long lastCommittedLength = validLength; + boolean openEventGroup = false; + UUID openEventGroupId = null; + int nextOperationIndex = 0; + long expectedSequence = -1; + boolean incompleteTail = false; + Footer footer = null; + + while (reader.getFilePointer() < fileLength) { + final long frameOffset = reader.getFilePointer(); + if (fileLength - frameOffset < Integer.BYTES) { + incompleteTail = true; + break; + } + + final int magic = reader.readInt(); + reader.seek(frameOffset); + if (magic == LogicalBackupFormat.FOOTER_MAGIC) { + if (fileLength - frameOffset < LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + incompleteTail = true; + break; + } + if (fileLength - frameOffset > LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278, + segment)); + } + final byte[] footerBytes = readBytes(reader, LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + footer = + parseAndValidateFooter( + footerBytes, + segmentId, + createdAt, + records.isEmpty() ? -1 : records.get(0).getSequence(), + records.isEmpty() ? -1 : records.get(records.size() - 1).getSequence(), + frameOffset, + records.size(), + digest.digest(), + segment); + validLength = reader.getFilePointer(); + lastCommittedLength = validLength; + break; + } + if (magic != LogicalBackupFormat.RECORD_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5, + frameOffset, + segment)); + } + if (fileLength - frameOffset < LogicalBackupFormat.RECORD_HEADER_SIZE) { + incompleteTail = true; + break; + } + + final byte[] header = readBytes(reader, LogicalBackupFormat.RECORD_HEADER_SIZE); + final ByteBuffer headerBuffer = wrap(header); + headerBuffer.getInt(); + final byte recordTypeCode = headerBuffer.get(); + final long sequence = headerBuffer.getLong(); + final UUID eventGroupId = new UUID(headerBuffer.getLong(), headerBuffer.getLong()); + final int operationIndex = headerBuffer.getInt(); + final long eventTime = headerBuffer.getLong(); + final byte requestVersion = headerBuffer.get(); + final short requestType = headerBuffer.getShort(); + final int metadataLength = headerBuffer.getInt(); + final int payloadLength = headerBuffer.getInt(); + final int headerCrc = headerBuffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79, + frameOffset, + segment)); + } + if (metadataLength < 0 + || payloadLength < 0 + || metadataLength > maxRecordBytes + || payloadLength > maxRecordBytes + || (long) metadataLength + payloadLength > maxRecordBytes) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E, + frameOffset, + segment)); + } + final long remainingLength = (long) metadataLength + payloadLength + Integer.BYTES; + if (fileLength - reader.getFilePointer() < remainingLength) { + incompleteTail = true; + break; + } + + final byte[] metadataBytes = readBytes(reader, metadataLength); + final byte[] payload = readBytes(reader, payloadLength); + final int frameCrc = reader.readInt(); + final byte[] frameContent = new byte[metadataLength + payloadLength]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataLength); + System.arraycopy(payload, 0, frameContent, metadataLength, payloadLength); + if (frameCrc != LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552, + frameOffset, + segment)); + } + + final LogicalBackupRecordType recordType = LogicalBackupRecordType.valueOf(recordTypeCode); + if (recordType == null) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D, + recordTypeCode, + segment)); + } + if (expectedSequence >= 0 && sequence != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B, + frameOffset, + segment)); + } + final LogicalBackupRecord record = + new LogicalBackupRecord( + recordType, + sequence, + eventGroupId, + operationIndex, + eventTime, + requestVersion, + requestType, + new String(metadataBytes, StandardCharsets.UTF_8), + payload); + records.add(record); + digest.update(header); + digest.update(frameContent); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(frameCrc).array()); + validLength = reader.getFilePointer(); + if (recordType == LogicalBackupRecordType.EVENT_BEGIN) { + if (openEventGroup || operationIndex != -1) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A, + segment)); + } + openEventGroup = true; + openEventGroupId = eventGroupId; + nextOperationIndex = 0; + } else if (recordType == LogicalBackupRecordType.EVENT_COMMIT) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + openEventGroup = false; + openEventGroupId = null; + lastCommittedLength = validLength; + } else if (recordType == LogicalBackupRecordType.PIPE_REQUEST) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F, + frameOffset, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + nextOperationIndex++; + } else if (openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B, + frameOffset, + segment)); + } else if (!openEventGroup) { + lastCommittedLength = validLength; + } + expectedSequence = sequence + 1; + } + + if (incompleteTail && !allowIncompleteTail) { + throw new EOFException( + String.format( + LogicalBackupMessages.EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7, + segment)); + } + if (footer != null && openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8, + segment)); + } + return new ScanResult( + segmentId, + createdAt, + records, + footer, + validLength, + lastCommittedLength, + incompleteTail, + openEventGroup); + } + } + + private static void validateSegmentHeader(final byte[] header, final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(header); + if (buffer.getLong() != LogicalBackupFormat.SEGMENT_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73, + segment)); + } + final short majorVersion = buffer.getShort(); + buffer.getShort(); + if (majorVersion != LogicalBackupFormat.MAJOR_VERSION) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB, + majorVersion, + segment)); + } + buffer.position(LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES); + final int headerCrc = buffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55, + segment)); + } + } + + private static Footer parseAndValidateFooter( + final byte[] bytes, + final long expectedSegmentId, + final long expectedCreatedAt, + final long expectedFirstSequence, + final long expectedLastSequence, + final long expectedValidBytes, + final long expectedRecordCount, + final byte[] expectedDigest, + final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(bytes); + if (buffer.getInt() != LogicalBackupFormat.FOOTER_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4, + segment)); + } + final long segmentId = buffer.getLong(); + final long firstSequence = buffer.getLong(); + final long lastSequence = buffer.getLong(); + final long recordCount = buffer.getLong(); + final long validBytes = buffer.getLong(); + final long createdAt = buffer.getLong(); + final long sealedAt = buffer.getLong(); + final byte[] digest = new byte[LogicalBackupFormat.SHA256_SIZE]; + buffer.get(digest); + final int footerCrc = buffer.getInt(); + if (segmentId != expectedSegmentId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D, + segment)); + } + if (createdAt != expectedCreatedAt + || firstSequence != expectedFirstSequence + || lastSequence != expectedLastSequence + || validBytes != expectedValidBytes + || recordCount != expectedRecordCount) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB, + segment)); + } + if (!Arrays.equals(digest, expectedDigest)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116, + segment)); + } + if (footerCrc + != LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91, + segment)); + } + return new Footer( + segmentId, + firstSequence, + lastSequence, + recordCount, + validBytes, + createdAt, + sealedAt, + digest); + } + + private static byte[] readBytes(final RandomAccessFile reader, final int length) + throws IOException { + final byte[] bytes = new byte[length]; + reader.readFully(bytes); + return bytes; + } + + private static ByteBuffer wrap(final byte[] bytes) { + return ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + } + + public static class ScanResult { + private final long segmentId; + private final long createdAt; + private final List records; + private final Footer footer; + private final long validLength; + private final long lastCommittedLength; + private final boolean incompleteTail; + private final boolean openEventGroup; + + private ScanResult( + final long segmentId, + final long createdAt, + final List records, + final Footer footer, + final long validLength, + final long lastCommittedLength, + final boolean incompleteTail, + final boolean openEventGroup) { + this.segmentId = segmentId; + this.createdAt = createdAt; + this.records = Collections.unmodifiableList(records); + this.footer = footer; + this.validLength = validLength; + this.lastCommittedLength = lastCommittedLength; + this.incompleteTail = incompleteTail; + this.openEventGroup = openEventGroup; + } + + public long getSegmentId() { + return segmentId; + } + + public long getCreatedAt() { + return createdAt; + } + + public List getRecords() { + return records; + } + + public Footer getFooter() { + return footer; + } + + public long getValidLength() { + return validLength; + } + + public long getLastCommittedLength() { + return lastCommittedLength; + } + + public boolean hasIncompleteTail() { + return incompleteTail; + } + + public boolean hasOpenEventGroup() { + return openEventGroup; + } + + public boolean isSealed() { + return footer != null; + } + } + + public static class Footer { + private final long segmentId; + private final long firstSequence; + private final long lastSequence; + private final long recordCount; + private final long validBytes; + private final long createdAt; + private final long sealedAt; + private final byte[] digest; + + private Footer( + final long segmentId, + final long firstSequence, + final long lastSequence, + final long recordCount, + final long validBytes, + final long createdAt, + final long sealedAt, + final byte[] digest) { + this.segmentId = segmentId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.recordCount = recordCount; + this.validBytes = validBytes; + this.createdAt = createdAt; + this.sealedAt = sealedAt; + this.digest = digest; + } + + public long getSegmentId() { + return segmentId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public long getRecordCount() { + return recordCount; + } + + public long getValidBytes() { + return validBytes; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getSealedAt() { + return sealedAt; + } + + public byte[] getDigest() { + return digest; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java new file mode 100644 index 0000000000000..5a95830d03e8a --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +/** Settings shared by ConfigRegion, SchemaRegion and DataRegion backup writers. */ +public final class LogicalBackupSettings { + + public final Path root; + public final String backupId; + public final String resume; + public final long segmentSize; + public final int maxRecord; + public final LogicalBackupWriter.FsyncPolicy fsync; + public final int fsyncBatch; + public final long fsyncPeriod; + public final boolean includeHeartbeat; + public final String unsupported; + public final Map captureOptions; + + public LogicalBackupSettings(final PipeParameters parameters, final String defaultBackupId) + throws IOException { + try { + final String directory = value(parameters, "dir", ""); + if (directory.isBlank()) { + throw invalid("dir"); + } + root = Path.of(directory).toAbsolutePath().normalize(); + LogicalBackupCatalog.rejectSymbolicLinks(root); + backupId = value(parameters, "backup-id", defaultBackupId); + validateBackupId(backupId); + resume = value(parameters, "resume", "fail-if-exists").toLowerCase(Locale.ROOT); + if (!List.of("fail-if-exists", "append", "new").contains(resume)) { + throw invalid("resume"); + } + segmentSize = + Long.parseLong( + value( + parameters, + "segment-size-bytes", + Long.toString(PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE))); + maxRecord = + Integer.parseInt( + value( + parameters, + "max-record-bytes", + Integer.toString( + PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE))); + fsync = + LogicalBackupWriter.FsyncPolicy.valueOf( + value(parameters, "fsync-policy", "always").toUpperCase(Locale.ROOT)); + fsyncBatch = Integer.parseInt(value(parameters, "fsync-batch-operations", "1000")); + fsyncPeriod = Long.parseLong(value(parameters, "fsync-period-ms", "1000")); + includeHeartbeat = booleanValue(parameters, "include-heartbeat"); + unsupported = value(parameters, "on-unsupported-event", "fail"); + if (maxRecord <= 0 + || maxRecord > LogicalBackupFormat.MAX_RECORD_BYTES + || segmentSize < maxRecord + || fsyncBatch <= 0 + || fsyncPeriod <= 0 + || !List.of("fail", "skip").contains(unsupported) + || (has(parameters, "fsync-batch-operations") + && fsync != LogicalBackupWriter.FsyncPolicy.BATCH) + || (has(parameters, "fsync-period-ms") + && fsync != LogicalBackupWriter.FsyncPolicy.PERIODIC)) { + throw invalid("writer"); + } + captureOptions = + new Gson() + .fromJson( + value(parameters, "capture-options", "{}"), + new TypeToken>() {}.getType()); + if (captureOptions == null) { + throw invalid("capture-options"); + } + if (!List.of( + "scope", + "model", + "mode", + "include-auth", + "include", + "exclude", + "start-time", + "end-time", + "until", + "segment-size-bytes", + "max-record-bytes", + "fsync-policy", + "on-unsupported", + "include-heartbeat", + "fsync-batch-operations", + "fsync-period-ms") + .containsAll(captureOptions.keySet()) + || captureOptions.containsValue(null)) { + throw invalid("capture-options"); + } + } catch (final IllegalArgumentException | com.google.gson.JsonParseException e) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB, e); + } + } + + public void requireSharedResume(final PipeParameters parameters) throws IOException { + if (!has(parameters, "resume") || !"append".equals(resume)) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90); + } + } + + public LogicalBackupWriter open( + final String streamType, final int regionId, final LogicalBackupManifest manifest) + throws IOException { + Path directory = root.resolve(backupId).resolve(streamType + "-" + regionId); + if ("new".equals(resume)) { + directory = directory.resolveSibling(directory.getFileName() + "-new-" + UUID.randomUUID()); + } + manifest.captureOptions = new LinkedHashMap<>(captureOptions); + return new LogicalBackupWriter( + directory, + manifest, + segmentSize, + maxRecord, + fsync, + fsyncBatch, + fsyncPeriod, + "append".equals(resume)); + } + + public static void validateBackupId(final String value) throws IOException { + if (value == null + || !value.matches("[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}") + || value.endsWith(".") + || value.matches("(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\\..*)?")) { + throw invalid("backup-id"); + } + } + + private static boolean booleanValue(final PipeParameters parameters, final String key) + throws IOException { + final String value = value(parameters, key, "false"); + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw invalid(key); + } + return Boolean.parseBoolean(value); + } + + private static boolean has(final PipeParameters parameters, final String key) { + return parameters.hasAnyAttributes("connector." + key, "sink." + key); + } + + private static String value( + final PipeParameters parameters, final String key, final String fallback) throws IOException { + if (parameters.getAttribute().containsKey("connector." + key) + && parameters.getAttribute().containsKey("sink." + key)) { + throw invalid(key); + } + return parameters.getStringOrDefault( + Arrays.asList("connector." + key, "sink." + key), fallback); + } + + private static IOException invalid(final String key) { + return new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB, key)); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java new file mode 100644 index 0000000000000..86536587efef9 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -0,0 +1,1181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +public class LogicalBackupWriter implements Closeable { + + public enum FsyncPolicy { + ALWAYS, + BATCH, + PERIODIC, + NONE + } + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private final Path directory; + private final Path manifestPath; + private final Path lockPath; + private final int maxRecordBytes; + private final long segmentSizeBytes; + private final FsyncPolicy fsyncPolicy; + private final int fsyncBatchOperations; + private final long fsyncPeriodMs; + private final LogicalBackupManifest manifest; + private final LogicalBackupSegmentReader reader; + + private FileChannel channel; + private FileChannel lockChannel; + private FileLock directoryLock; + private long segmentId; + private long segmentCreatedAt; + private long segmentFirstSequence = -1; + private long segmentLastSequence = -1; + private long segmentRecordCount; + private MessageDigest segmentDigest; + private long nextSequence; + private int operationsSinceFsync; + private long lastFsyncAt; + private boolean closed; + private boolean streamEnded; + private boolean recoveredDuringOpen; + + public LogicalBackupWriter( + final Path directory, + final LogicalBackupManifest manifest, + final long segmentSizeBytes, + final int maxRecordBytes, + final FsyncPolicy fsyncPolicy, + final int fsyncBatchOperations, + final long fsyncPeriodMs, + final boolean append) + throws IOException { + this.directory = + Objects.requireNonNull(directory, LogicalBackupMessages.EXCEPTION_DIRECTORY_5F8F22B8) + .toAbsolutePath() + .normalize(); + this.manifest = + Objects.requireNonNull(manifest, LogicalBackupMessages.EXCEPTION_MANIFEST_7F5CB74A); + this.segmentSizeBytes = segmentSizeBytes; + this.maxRecordBytes = maxRecordBytes; + this.fsyncPolicy = + Objects.requireNonNull(fsyncPolicy, LogicalBackupMessages.EXCEPTION_FSYNC_POLICY_6D493614); + if (segmentSizeBytes + <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + || maxRecordBytes <= 0 + || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || segmentSizeBytes < maxRecordBytes + || fsyncBatchOperations <= 0 + || fsyncPeriodMs <= 0) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99); + } + this.fsyncBatchOperations = fsyncBatchOperations; + this.fsyncPeriodMs = fsyncPeriodMs; + this.manifestPath = this.directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + this.lockPath = this.directory.resolve(".backup.lock"); + this.reader = new LogicalBackupSegmentReader(maxRecordBytes); + try { + prepareDirectory(append); + acquireLock(); + if (Files.isSymbolicLink(manifestPath)) { + throw symbolicLinkException(manifestPath); + } + if (Files.exists(manifestPath)) { + if (!append) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory)); + } + loadExistingManifest(); + } else { + manifest.status = "WRITING"; + manifest.createdAt = Instant.now().toString(); + manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT); + manifest.fsyncBatchOperations = fsyncBatchOperations; + manifest.fsyncPeriodMs = fsyncPeriodMs; + manifest.segmentSizeBytes = segmentSizeBytes; + manifest.maxRecordBytes = maxRecordBytes; + writeManifest(); + } + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + nextSequence = Math.max(0, manifest.lastSequence + 1); + openOrCreateSegment(); + rebuildManifestFromSegments(); + if (recoveredDuringOpen) { + manifest.lastDurableSequence = manifest.lastSequence; + } else { + manifest.lastDurableSequence = + Math.max(-1, Math.min(manifest.lastDurableSequence, manifest.lastSequence)); + } + updateActiveSegmentManifest(); + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + writeManifest(); + lastFsyncAt = System.currentTimeMillis(); + } catch (final IOException | RuntimeException e) { + if (channel != null) { + try { + channel.close(); + } catch (final IOException closeException) { + e.addSuppressed(closeException); + } + channel = null; + } + releaseLock(); + throw e; + } + } + + private void prepareDirectory(final boolean append) throws IOException { + rejectSymbolicLinksInDirectoryPath(); + if (append) { + Files.createDirectories(directory); + rejectSymbolicLinksInDirectoryPath(); + return; + } + final Path parent = directory.getParent(); + if (parent != null) { + Files.createDirectories(parent); + rejectSymbolicLinksInDirectoryPath(); + } + try { + Files.createDirectory(directory); + } catch (final FileAlreadyExistsException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory), + e); + } + rejectSymbolicLinksInDirectoryPath(); + } + + private void rejectSymbolicLinksInDirectoryPath() throws IOException { + Path current = directory.getRoot(); + for (final Path part : directory) { + current = current == null ? part : current.resolve(part); + if (Files.isSymbolicLink(current)) { + throw symbolicLinkException(current); + } + } + } + + private static IOException symbolicLinkException(final Path path) { + return new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569, + path)); + } + + public synchronized long writeEvent( + final UUID eventGroupId, + final long eventTime, + final List requests, + final String metadata) + throws IOException { + ensureOpen(); + if (requests == null || requests.isEmpty()) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC); + } + Objects.requireNonNull(eventGroupId, LogicalBackupMessages.EXCEPTION_EVENT_GROUP_ID_C6F6268A); + final String safeMetadata = metadata == null ? "" : metadata; + final String eventDigest = computeEventDigest(requests); + if (eventGroupId.toString().equals(manifest.lastEventGroupId)) { + if (eventDigest.equals(manifest.lastEventDigest)) { + updateActiveSegmentManifest(); + writeManifest(); + return manifest.lastEventFirstSequence; + } + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330, + eventGroupId)); + } + rollBeforeEventIfNecessary(requests, safeMetadata); + final long eventStartPosition = channel.position(); + final long firstSequence = nextSequence; + try { + appendRecord( + LogicalBackupRecordType.EVENT_BEGIN, eventGroupId, -1, eventTime, null, safeMetadata); + int operationIndex = 0; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + appendRecord( + LogicalBackupRecordType.PIPE_REQUEST, + eventGroupId, + operationIndex++, + eventTime, + request, + safeMetadata); + } + appendRecord( + LogicalBackupRecordType.EVENT_COMMIT, + eventGroupId, + operationIndex, + eventTime, + null, + safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(eventStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = eventDigest; + manifest.lastEventFirstSequence = firstSequence; + writeManifest(); + return firstSequence; + } + + public synchronized void writeControl( + final LogicalBackupRecordType recordType, final long eventTime, final String metadata) + throws IOException { + ensureOpen(); + final String safeMetadata = metadata == null ? "" : metadata; + rollBeforeControlIfNecessary(safeMetadata); + final long recordStartPosition = channel.position(); + try { + appendRecord(recordType, new UUID(0, 0), -1, eventTime, null, safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(recordStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + writeManifest(); + } + + public synchronized void recordSkippedEvent(final long eventTime, final String metadata) + throws IOException { + writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata); + manifest.skippedEventCount++; + writeManifest(); + } + + public synchronized LogicalBackupManifest getManifest() { + return manifest; + } + + public synchronized void heartbeat() throws IOException { + ensureOpen(); + if (!Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) + || directoryLock == null + || !directoryLock.isValid()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD, + directory)); + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + updateActiveSegmentManifest(); + writeManifest(); + } + } + + private void appendRecord( + final LogicalBackupRecordType recordType, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final TPipeTransferReq request, + final String metadata) + throws IOException { + final byte[] metadataBytes = metadata.getBytes(StandardCharsets.UTF_8); + final byte[] payload = + request == null + ? new byte[0] + : java.util.Arrays.copyOf(request.getBody(), request.getBody().length); + if ((long) metadataBytes.length + payload.length > maxRecordBytes) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996); + } + final long requiredBytes = + LogicalBackupFormat.RECORD_HEADER_SIZE + + metadataBytes.length + + payload.length + + Integer.BYTES; + final long sequence = nextSequence++; + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.RECORD_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putInt(LogicalBackupFormat.RECORD_MAGIC); + header.put(recordType.getCode()); + header.putLong(sequence); + header.putLong(eventGroupId.getMostSignificantBits()); + header.putLong(eventGroupId.getLeastSignificantBits()); + header.putInt(operationIndex); + header.putLong(eventTime); + header.put(request == null ? (byte) 0 : request.getVersion()); + header.putShort(request == null ? (short) 0 : request.getType()); + header.putInt(metadataBytes.length); + header.putInt(payload.length); + header.putInt(0); + final byte[] headerBytes = header.array(); + final int headerCrc = + LogicalBackupFormat.crc32c(headerBytes, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4); + ByteBuffer.wrap(headerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt(LogicalBackupFormat.RECORD_HEADER_SIZE - 4, headerCrc); + final byte[] frameContent = new byte[metadataBytes.length + payload.length]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataBytes.length); + System.arraycopy(payload, 0, frameContent, metadataBytes.length, payload.length); + final int frameCrc = LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length); + + writeFully(ByteBuffer.wrap(headerBytes)); + writeFully(ByteBuffer.wrap(frameContent)); + writeFully( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).flip()); + segmentDigest.update(headerBytes); + segmentDigest.update(frameContent); + segmentDigest.update( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).array()); + if (segmentFirstSequence < 0) { + segmentFirstSequence = sequence; + } + segmentLastSequence = sequence; + segmentRecordCount++; + manifest.firstSequence = manifest.firstSequence < 0 ? sequence : manifest.firstSequence; + manifest.lastSequence = sequence; + final String countKey = recordType.name(); + manifest.operationCounts.put(countKey, manifest.operationCounts.getOrDefault(countKey, 0L) + 1); + operationsSinceFsync++; + streamEnded = recordType == LogicalBackupRecordType.STREAM_END; + } + + private void openOrCreateSegment() throws IOException { + if (manifest.segments.isEmpty()) { + segmentId = 0; + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + final LogicalBackupManifest.Segment last = manifest.segments.get(manifest.segments.size() - 1); + final Path segmentPath = resolveExistingSegmentPath(last.file); + LogicalBackupSegmentReader.ScanResult scan = scanLastSegmentForAppend(segmentPath, last); + if (scan.isSealed()) { + if (!"SEALED".equals(last.status)) { + manifest.recovered = true; + recoveredDuringOpen = true; + } + segmentId = scan.getSegmentId() + 1; + // Validate every existing sealed segment before creating a new file in the backup. + rebuildManifestFromSegments(); + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + validateSegmentFiles(null); + if (scan.hasIncompleteTail() || scan.hasOpenEventGroup()) { + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(scan.getLastCommittedLength()); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + scan = reader.scan(segmentPath, false); + } + if (!scan.getRecords().isEmpty()) { + nextSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence() + 1; + manifest.lastSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence(); + } + segmentId = scan.getSegmentId(); + openExistingSegment(segmentPath, scan); + } + + private LogicalBackupSegmentReader.ScanResult scanLastSegmentForAppend( + final Path segmentPath, final LogicalBackupManifest.Segment segment) throws IOException { + try { + return reader.scan(segmentPath, true); + } catch (final IOException scanException) { + final long recoverableLength = + "SEALED".equals(segment.status) + ? -1 + : findRecoverableTailStart(segmentPath, segment.sizeBytes); + if (recoverableLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw scanException; + } + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(recoverableLength); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + return reader.scan(segmentPath, false); + } + } + + private static long findRecoverableTailStart( + final Path segmentPath, final long manifestSegmentSize) throws IOException { + final long fileSize = Files.size(segmentPath); + if (fileSize + >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + final ByteBuffer magic = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN); + try (final FileChannel readChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + readChannel.position(fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + while (magic.hasRemaining()) { + if (readChannel.read(magic) < 0) { + break; + } + } + } + if (!magic.hasRemaining() && magic.flip().getInt() == LogicalBackupFormat.FOOTER_MAGIC) { + return fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + } + } + if (manifestSegmentSize >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + && manifestSegmentSize < fileSize + && fileSize - manifestSegmentSize <= LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + return manifestSegmentSize; + } + return -1; + } + + private void openExistingSegment( + final Path segmentPath, final LogicalBackupSegmentReader.ScanResult scan) throws IOException { + channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ); + channel.position(channel.size()); + segmentCreatedAt = scan.getCreatedAt(); + segmentDigest = LogicalBackupFormat.newSha256(); + try (final FileChannel digestChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + final ByteBuffer digestBuffer = ByteBuffer.allocate(64 * 1024); + long remaining = scan.getValidLength(); + while (remaining > 0) { + digestBuffer.clear(); + digestBuffer.limit((int) Math.min(digestBuffer.capacity(), remaining)); + final int read = digestChannel.read(digestBuffer); + if (read < 0) { + break; + } + remaining -= read; + segmentDigest.update(digestBuffer.array(), 0, read); + } + } + segmentFirstSequence = -1; + segmentLastSequence = -1; + segmentRecordCount = 0; + for (final LogicalBackupRecord record : scan.getRecords()) { + segmentFirstSequence = segmentFirstSequence < 0 ? record.getSequence() : segmentFirstSequence; + segmentLastSequence = record.getSequence(); + segmentRecordCount++; + } + streamEnded = + !scan.getRecords().isEmpty() + && scan.getRecords().get(scan.getRecords().size() - 1).getRecordType() + == LogicalBackupRecordType.STREAM_END; + } + + private void openNewSegment() throws IOException { + final String fileName = segmentFileName(segmentId); + final Path segmentPath = resolveSegmentPath(fileName); + final Path temporarySegmentPath = resolveSegmentPath(fileName + ".tmp"); + if (Files.exists(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + adoptOrphanSegment(resolveExistingSegmentPath(fileName), fileName); + return; + } + + // A temporary segment is never exposed through the manifest and contains only a header, so it + // is safe to discard after an interrupted creation attempt. + Files.deleteIfExists(temporarySegmentPath); + segmentCreatedAt = System.currentTimeMillis(); + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(segmentCreatedAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4, + LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)); + try (final FileChannel temporaryChannel = + FileChannel.open( + temporarySegmentPath, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + StandardOpenOption.READ)) { + final ByteBuffer headerBuffer = ByteBuffer.wrap(bytes); + while (headerBuffer.hasRemaining()) { + temporaryChannel.write(headerBuffer); + } + temporaryChannel.force(true); + } + try { + Files.move(temporarySegmentPath, segmentPath, StandardCopyOption.ATOMIC_MOVE); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporarySegmentPath, segmentPath); + } + + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void adoptOrphanSegment(final Path segmentPath, final String fileName) + throws IOException { + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segmentId + || scan.isSealed() + || !scan.getRecords().isEmpty() + || scan.getValidLength() != LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentPath)); + } + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void addActiveSegmentToManifest(final String fileName) throws IOException { + final LogicalBackupManifest.Segment segment = new LogicalBackupManifest.Segment(); + segment.file = fileName; + segment.segmentId = segmentId; + segment.status = "ACTIVE"; + manifest.segments.add(segment); + writeManifest(); + } + + private void validateSegmentFiles(final String allowedOrphanFile) throws IOException { + final Set listedFiles = new HashSet<>(); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + listedFiles.add(segment.file); + } + try (final DirectoryStream segmentFiles = Files.newDirectoryStream(directory, "*.pwal")) { + for (final Path segmentFile : segmentFiles) { + final String fileName = segmentFile.getFileName().toString(); + if (!listedFiles.contains(fileName) && !fileName.equals(allowedOrphanFile)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentFile)); + } + } + } + } + + private static String segmentFileName(final long id) { + return String.format(Locale.ROOT, "segment-%020d.pwal", id); + } + + private void sealSegment() throws IOException { + if (channel == null || segmentRecordCount == 0) { + return; + } + final byte[] digest = segmentDigest.digest(); + final ByteBuffer footer = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN); + footer.putInt(LogicalBackupFormat.FOOTER_MAGIC); + footer.putLong(segmentId); + footer.putLong(segmentFirstSequence); + footer.putLong(segmentLastSequence); + footer.putLong(segmentRecordCount); + footer.putLong(channel.position()); + footer.putLong(segmentCreatedAt); + footer.putLong(System.currentTimeMillis()); + footer.put(digest); + footer.putInt(0); + final byte[] footerBytes = footer.array(); + ByteBuffer.wrap(footerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4, + LogicalBackupFormat.crc32c( + footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)); + writeFully(ByteBuffer.wrap(footerBytes)); + channel.force(true); + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.sha256 = LogicalBackupFormat.toHex(digest); + segment.status = "SEALED"; + manifest.lastDurableSequence = segmentLastSequence; + channel.close(); + channel = null; + writeManifest(); + } + + private boolean forceIfNeeded() throws IOException { + if (channel == null) { + return false; + } + final long now = System.currentTimeMillis(); + final boolean shouldForce = + fsyncPolicy == FsyncPolicy.ALWAYS + || (fsyncPolicy == FsyncPolicy.BATCH && operationsSinceFsync >= fsyncBatchOperations) + || (fsyncPolicy == FsyncPolicy.PERIODIC && now - lastFsyncAt >= fsyncPeriodMs); + if (shouldForce) { + channel.force(true); + operationsSinceFsync = 0; + lastFsyncAt = now; + } + return shouldForce; + } + + private void writeManifest() throws IOException { + final Path temporary = + manifestPath.resolveSibling(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp"); + final byte[] manifestBytes = GSON.toJson(manifest).getBytes(StandardCharsets.UTF_8); + Files.deleteIfExists(temporary); + try (final FileChannel manifestChannel = + FileChannel.open(temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + final ByteBuffer manifestBuffer = ByteBuffer.wrap(manifestBytes); + while (manifestBuffer.hasRemaining()) { + manifestChannel.write(manifestBuffer); + } + manifestChannel.force(true); + } + try { + Files.move( + temporary, + manifestPath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporary, manifestPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + private void loadExistingManifest() throws IOException { + final LogicalBackupManifest existing; + try { + existing = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + if (existing == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (!LogicalBackupFormat.FORMAT_NAME.equals(existing.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(existing.formatVersion)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E, + manifestPath)); + } + validateManifestIdentity("backupId", manifest.backupId, existing.backupId); + validateManifestIdentity("streamId", manifest.streamId, existing.streamId); + validateManifestIdentity("pipeName", manifest.pipeName, existing.pipeName); + validateManifestIdentity("streamType", manifest.streamType, existing.streamType); + validateManifestIdentity("sourceClusterId", manifest.sourceClusterId, existing.sourceClusterId); + validateManifestIdentity("sourceVersion", manifest.sourceVersion, existing.sourceVersion); + validateManifestIdentity( + "timestampPrecision", manifest.timestampPrecision, existing.timestampPrecision); + validateManifestIdentity("sinkTaskId", manifest.sinkTaskId, existing.sinkTaskId); + validateManifestIdentity( + "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy); + validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes); + validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes); + if (existing.fsyncBatchOperations > 0) { + validateManifestConfiguration( + "fsyncBatchOperations", fsyncBatchOperations, existing.fsyncBatchOperations); + } + if (existing.fsyncPeriodMs > 0) { + validateManifestConfiguration("fsyncPeriodMs", fsyncPeriodMs, existing.fsyncPeriodMs); + } + validateManifestIdentity( + "captureOptions", + GSON.toJson(manifest.captureOptions), + GSON.toJson(existing.captureOptions)); + if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "pipeCreationTime", + manifest.pipeCreationTime, + existing.pipeCreationTime)); + } + if (manifest.regionId != existing.regionId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "regionId", + manifest.regionId, + existing.regionId)); + } + manifest.status = existing.status; + manifest.backupId = existing.backupId; + manifest.pipeName = existing.pipeName; + manifest.pipeCreationTime = existing.pipeCreationTime; + manifest.sourceClusterId = existing.sourceClusterId; + manifest.sourceVersion = existing.sourceVersion; + manifest.timestampPrecision = existing.timestampPrecision; + manifest.streamId = existing.streamId; + manifest.regionId = existing.regionId; + manifest.sinkTaskId = existing.sinkTaskId; + manifest.streamType = existing.streamType; + manifest.firstSequence = existing.firstSequence; + manifest.lastSequence = existing.lastSequence; + manifest.lastDurableSequence = existing.lastDurableSequence; + manifest.recovered = existing.recovered; + manifest.segments = existing.segments == null ? new ArrayList<>() : existing.segments; + manifest.operationCounts = + existing.operationCounts == null + ? new java.util.LinkedHashMap<>() + : existing.operationCounts; + manifest.createdAt = existing.createdAt; + manifest.closedAt = existing.closedAt; + manifest.sealedAt = existing.sealedAt; + manifest.captureOptions = existing.captureOptions; + manifest.fsyncPolicy = existing.fsyncPolicy; + manifest.fsyncBatchOperations = fsyncBatchOperations; + manifest.fsyncPeriodMs = fsyncPeriodMs; + manifest.segmentSizeBytes = existing.segmentSizeBytes; + manifest.maxRecordBytes = existing.maxRecordBytes; + manifest.lastEventGroupId = existing.lastEventGroupId; + manifest.lastEventDigest = existing.lastEventDigest; + manifest.lastEventFirstSequence = existing.lastEventFirstSequence; + manifest.skippedEventCount = existing.skippedEventCount; + } + + private void validateManifestIdentity( + final String field, final String expected, final String actual) throws IOException { + if (expected != null && !Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void validateManifestConfiguration( + final String field, final long expected, final long actual) throws IOException { + if (expected != actual) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void acquireLock() throws IOException { + try { + if (Files.isSymbolicLink(lockPath)) { + throw symbolicLinkException(lockPath); + } + lockChannel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + directoryLock = lockChannel.tryLock(); + if (directoryLock == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory)); + } + lockChannel.truncate(0); + lockChannel.write( + StandardCharsets.UTF_8.encode(Long.toString(ProcessHandle.current().pid()))); + lockChannel.force(true); + } catch (final OverlappingFileLockException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory), + e); + } + } + + private void releaseLock() { + try { + if (directoryLock != null) { + directoryLock.release(); + directoryLock = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + try { + if (lockChannel != null) { + lockChannel.close(); + lockChannel = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + } + + private void writeFully(final ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } + + private void ensureOpen() throws IOException { + if (closed || channel == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB); + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + try { + if (channel != null) { + if (!streamEnded) { + writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), ""); + } + sealSegment(); + } + manifest.status = manifest.skippedEventCount > 0 ? "INCOMPLETE" : "SEALED"; + manifest.closedAt = Instant.now().toString(); + manifest.sealedAt = manifest.closedAt; + manifest.lastDurableSequence = manifest.lastSequence; + writeManifest(); + } finally { + if (channel != null) { + channel.close(); + channel = null; + } + closed = true; + releaseLock(); + } + } + + private void rollBeforeEventIfNecessary( + final List requests, final String metadata) throws IOException { + final long metadataBytes = metadata.getBytes(StandardCharsets.UTF_8).length; + long requiredBytes = frameSize(metadataBytes, 0) * 2; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + requiredBytes += frameSize(metadataBytes, request.getBody().length); + } + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private void rollBeforeControlIfNecessary(final String metadata) throws IOException { + final long requiredBytes = frameSize(metadata.getBytes(StandardCharsets.UTF_8).length, 0); + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private static long frameSize(final long metadataBytes, final long payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String computeEventDigest(final List requests) + throws IOException { + final MessageDigest digest = LogicalBackupFormat.newSha256(); + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + updateEventDigest(digest, request.getVersion(), request.getType(), request.getBody()); + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + private static void updateEventDigest( + final MessageDigest digest, + final byte requestVersion, + final short requestType, + final byte[] body) { + digest.update(requestVersion); + digest.update( + ByteBuffer.allocate(Short.BYTES).order(ByteOrder.BIG_ENDIAN).putShort(requestType).array()); + digest.update(body); + } + + private void rollbackTo(final long position) throws IOException { + final Path segmentPath = currentSegmentPath(); + channel.truncate(position); + channel.force(true); + channel.close(); + channel = null; + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + rebuildManifestFromSegments(); + manifest.recovered = true; + manifest.status = "RECOVERED"; + manifest.lastDurableSequence = manifest.lastSequence; + operationsSinceFsync = 0; + lastFsyncAt = System.currentTimeMillis(); + updateActiveSegmentManifest(); + writeManifest(); + } + + private void rebuildManifestFromSegments() throws IOException { + manifest.firstSequence = -1; + manifest.lastSequence = -1; + manifest.lastEventGroupId = null; + manifest.lastEventDigest = null; + manifest.lastEventFirstSequence = -1; + manifest.skippedEventCount = 0; + manifest.operationCounts = new LinkedHashMap<>(); + + long expectedSequence = -1; + long expectedSegmentId = -1; + for (int segmentIndex = 0; segmentIndex < manifest.segments.size(); segmentIndex++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(segmentIndex); + final Path segmentPath = resolveExistingSegmentPath(segment.file); + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segment.segmentId) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if ((expectedSegmentId >= 0 && scan.getSegmentId() != expectedSegmentId) + || (!scan.isSealed() && segmentIndex != manifest.segments.size() - 1)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + expectedSegmentId = scan.getSegmentId() + 1; + if (scan.isSealed() && "SEALED".equals(segment.status)) { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + segment.firstSequence = -1; + segment.lastSequence = -1; + segment.recordCount = scan.getRecords().size(); + segment.sizeBytes = scan.getValidLength(); + segment.status = scan.isSealed() ? "SEALED" : "ACTIVE"; + segment.sha256 = + scan.isSealed() ? LogicalBackupFormat.toHex(scan.getFooter().getDigest()) : null; + + MessageDigest eventDigest = null; + UUID eventGroupId = null; + long eventFirstSequence = -1; + for (final LogicalBackupRecord record : scan.getRecords()) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149, + segmentPath)); + } + expectedSequence = record.getSequence() + 1; + segment.firstSequence = + segment.firstSequence < 0 ? record.getSequence() : segment.firstSequence; + segment.lastSequence = record.getSequence(); + manifest.firstSequence = + manifest.firstSequence < 0 ? record.getSequence() : manifest.firstSequence; + manifest.lastSequence = record.getSequence(); + manifest.operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + if (record.getRecordType() == LogicalBackupRecordType.SKIPPED_EVENT) { + manifest.skippedEventCount++; + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventDigest = LogicalBackupFormat.newSha256(); + eventGroupId = record.getEventGroupId(); + eventFirstSequence = record.getSequence(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + updateEventDigest( + eventDigest, + record.getRequestVersion(), + record.getRequestType(), + record.getPayload()); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = LogicalBackupFormat.toHex(eventDigest.digest()); + manifest.lastEventFirstSequence = eventFirstSequence; + eventDigest = null; + eventGroupId = null; + eventFirstSequence = -1; + } + } + } + nextSequence = Math.max(0, manifest.lastSequence + 1); + streamEnded = + manifest.lastSequence >= 0 + && !manifest.segments.isEmpty() + && isLastRecordStreamEnd(reader.scan(currentSegmentPath(), false).getRecords()); + } + + private static boolean isLastRecordStreamEnd(final List records) { + return !records.isEmpty() + && records.get(records.size() - 1).getRecordType() == LogicalBackupRecordType.STREAM_END; + } + + private Path currentSegmentPath() throws IOException { + if (manifest.segments.isEmpty()) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15); + } + return resolveSegmentPath(manifest.segments.get(manifest.segments.size() - 1).file); + } + + private Path resolveSegmentPath(final String file) throws IOException { + final Path relativePath; + try { + relativePath = file == null ? null : Path.of(file); + } catch (final RuntimeException e) { + throw invalidSegmentPath(file, e); + } + final Path normalizedDirectory = directory.toAbsolutePath().normalize(); + if (relativePath == null || relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + throw invalidSegmentPath(file, null); + } + final Path resolved = normalizedDirectory.resolve(relativePath).normalize(); + if (!resolved.startsWith(normalizedDirectory) + || !normalizedDirectory.equals(resolved.getParent())) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private Path resolveExistingSegmentPath(final String file) throws IOException { + final Path resolved = resolveSegmentPath(file); + if (Files.isSymbolicLink(resolved) + || !Files.isRegularFile(resolved, LinkOption.NOFOLLOW_LINKS)) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private static IOException invalidSegmentPath(final String file, final Throwable cause) { + final IOException exception = + new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + file)); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + + private void updateActiveSegmentManifest() throws IOException { + if (channel == null || manifest.segments.isEmpty()) { + return; + } + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.status = "ACTIVE"; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java index 8152ffe43e7d4..e52f6b101ce0a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java @@ -41,6 +41,20 @@ public Map getParams() { protected abstract PipeRequestType getPlanType(); + public static TPipeTransferReq toTPipeTransferReq( + final PipeRequestType requestType, final Map params) throws IOException { + final TPipeTransferReq request = new TPipeTransferReq(); + request.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); + request.type = requestType.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + writeParams(params, outputStream); + request.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + return request; + } + /////////////////////////////// Thrift /////////////////////////////// protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params) @@ -49,11 +63,7 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map entry : params.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue(), outputStream); - } + writeParams(params, outputStream); this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); } @@ -61,6 +71,15 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params, final DataOutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(params.size(), outputStream); + for (final Map.Entry entry : params.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + } + protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq( TPipeTransferReq transferReq) { final ByteBuffer bodyBuffer = transferReq.body.duplicate(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index d4d55ba7dac7a..7ef4cf1b52178 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -56,6 +56,8 @@ @TableModel public abstract class IoTDBNonDataRegionSource extends IoTDBSource { + public static final long LOGICAL_BACKUP_COMPLETED_INDEX = Long.MAX_VALUE - 1; + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBNonDataRegionSource.class); protected IoTDBTreePatternOperations treePattern; @@ -66,6 +68,14 @@ public abstract class IoTDBNonDataRegionSource extends IoTDBSource { // filled. Used only for metric framework. private int historicalEventsCount = 0; + // Logical backup snapshots must acknowledge all metadata before the coordinator drops the pipe. + // Leave room for queue maintenance to calculate index + 1 without overflowing. + private boolean finiteSnapshot; + private boolean snapshotOnly; + private long captureUntil; + private volatile long captureEndIndex = Long.MAX_VALUE; + private boolean snapshotCompletionReported; + private ConcurrentIterableLinkedQueue.DynamicIterator iterator; // If close() is called, hasBeenClosed will be set to true even if the extractor is started again. @@ -96,6 +106,9 @@ public void customize( } treePattern = (IoTDBTreePatternOperations) pattern; tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); + snapshotOnly = parameters.getBooleanOrDefault("source.logical-backup.snapshot", false); + captureUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L); + finiteSnapshot = snapshotOnly || captureUntil > 0; } @Override @@ -107,6 +120,13 @@ public void start() throws Exception { final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); + if (finiteSnapshot + && metaProgressIndex != null + && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) { + snapshotCompletionReported = true; + super.start(); + return; + } final long nextIndex = Objects.isNull(metaProgressIndex) // If the index is invalid, the queue is seen as cleared before and thus @@ -115,6 +135,12 @@ public void start() throws Exception { ? getNextIndexAfterSnapshot() : metaProgressIndex.getIndex() + 1; iterator = getListeningQueue().newIterator(nextIndex); + if (captureUntil > 0) { + java.util.concurrent.CompletableFuture.delayedExecutor( + Math.max(0L, captureUntil - System.currentTimeMillis()), + java.util.concurrent.TimeUnit.MILLISECONDS) + .execute(() -> captureEndIndex = getListeningQueue().getTailIndex()); + } super.start(); } @@ -171,6 +197,9 @@ public EnrichedEvent supply() throws Exception { return null; } } + if (snapshotCompletionReported) { + return null; + } // Check whether snapshot being parsed exists PipeWritePlanEvent realtimeEvent = lastEvent; @@ -197,7 +226,9 @@ public EnrichedEvent supply() throws Exception { Long.MIN_VALUE, Long.MAX_VALUE); - if (canSkipSnapshotPrivilegeCheck(historicalEvent)) { + // Portable backups contain logical metadata plans, even when the user can read the whole + // snapshot. Snapshot file transfer remains available to ordinary replication pipes. + if (!finiteSnapshot && canSkipSnapshotPrivilegeCheck(historicalEvent)) { if (historicalEvents.isEmpty()) { // We only report progress for the last snapshot event. historicalEvent.bindProgressIndex(new MetaProgressIndex(iterator.getNextIndex() - 1)); @@ -221,7 +252,20 @@ public EnrichedEvent supply() throws Exception { // Realtime if (Objects.isNull(realtimeEvent)) { + if (finiteSnapshot + && (snapshotOnly || captureUntil <= 0 || iterator.getNextIndex() >= captureEndIndex)) { + final ProgressReportEvent completion = + new ProgressReportEvent(pipeName, creationTime, pipeTaskMeta); + completion.bindProgressIndex(new MetaProgressIndex(LOGICAL_BACKUP_COMPLETED_INDEX)); + completion.increaseReferenceCount(IoTDBNonDataRegionSource.class.getName()); + snapshotCompletionReported = true; + return completion; + } realtimeEvent = (PipeWritePlanEvent) iterator.next(getMaxBlockingTimeMs()); + if (captureUntil > 0 && iterator.getNextIndex() > captureEndIndex) { + // A blocking read may wake with an event appended after the boundary was captured. + realtimeEvent = null; + } } if (Objects.isNull(realtimeEvent)) { return null; @@ -305,6 +349,11 @@ public long getUnTransferredEventCount() { final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); + if (finiteSnapshot + && metaProgressIndex != null + && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) { + return 0L; + } return Objects.nonNull(metaProgressIndex) ? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1 : getListeningQueue().getSize() + historicalEventsCount; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java new file mode 100644 index 0000000000000..3702850597d9e --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupArchiveReaderTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testReadCompleteArchiveAndOrderConfigThenSchemaThenData() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + writeStream(root.resolve("config"), "config-stream", "config", 4096); + + final List streams = new LogicalBackupArchiveReader().read(root, false); + + Assert.assertEquals(3, streams.size()); + Assert.assertEquals("config-stream", streams.get(0).getManifest().streamId); + Assert.assertEquals("schema-stream", streams.get(1).getManifest().streamId); + Assert.assertEquals("data-stream", streams.get(2).getManifest().streamId); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + Assert.assertEquals(4, streams.get(0).getRecords().size()); + Assert.assertEquals( + request(1), + streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq()); + } + + @Test + public void testManifestCounterMismatchIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.operationCounts.clear(); + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testUnlistedSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + Files.copy(onlySegment(directory), directory.resolve("unlisted.pwal")); + + assertReadFails(directory, false); + } + + @Test + public void testIncompleteOptInCannotBypassMissingFooter() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, false); + assertReadFails(directory, true); + } + + @Test + public void testUnsealedNonLastSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 500); + final List segments = segments(directory); + Assert.assertTrue(segments.size() > 1); + try (final RandomAccessFile file = new RandomAccessFile(segments.get(0).toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, true); + } + + @Test + public void testParentSegmentPathIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testSkippedEventRequiresExplicitIncompleteOptIn() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest("stream", "data"); + try (final LogicalBackupWriter writer = writer(directory, manifest, 4096)) { + writer.recordSkippedEvent(1, "unsupported-event"); + } + + assertReadFails(directory, false); + Assert.assertEquals(1, new LogicalBackupArchiveReader().read(directory, true).size()); + + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest persistedManifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + persistedManifest.skippedEventCount = 0; + Files.writeString(manifestPath, GSON.toJson(persistedManifest), StandardCharsets.UTF_8); + assertReadFails(directory, false); + } + + @Test + public void testMixedBackupSessionsAreRejected() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + final Path manifestPath = + root.resolve("schema").resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.backupId = "another-backup"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(root, false); + } + + @Test + public void testNonReplayableRequestTypeIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Collections.singletonList( + new TPipeTransferReq() + .setVersion((byte) 1) + .setType(Short.MAX_VALUE) + .setBody(ByteBuffer.wrap(new byte[] {1}))), + "metadata"); + } + + assertReadFails(directory, false); + } + + @Test + public void testConfigRequestTypesAreReplayable() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = + writer(directory, manifest("config-0", "config"), 4096)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Arrays.asList(request((short) 200), request((short) 201), request((short) 202)), + "metadata"); + } + + final List streams = new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(3, streams.get(0).getEventGroups().get(0).getRequests().size()); + } + + @Test + public void testDuplicateEventGroupIdIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventGroupId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent(eventGroupId, 1, Collections.singletonList(request(1)), "metadata"); + writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "metadata"); + writer.writeEvent(eventGroupId, 3, Collections.singletonList(request(1)), "metadata"); + } + + assertReadFails(directory, false); + } + + @Test + public void testRootManifestIsPortableAndOrdersAllStreams() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data-7"), "data-7", "data", 4096); + writeStream(root.resolve("schema-3"), "schema-3", "schema", 4096); + writeStream(root.resolve("config-0"), "config-0", "config", 4096); + final LogicalBackupCatalog catalog = sealCatalog(root); + + final List streams = + new LogicalBackupArchiveReader().read(root.resolve("manifest.json"), false); + Assert.assertEquals("config", streams.get(0).getManifest().streamType); + Assert.assertEquals("schema", streams.get(1).getManifest().streamType); + Assert.assertEquals("data", streams.get(2).getManifest().streamType); + Assert.assertEquals(3, catalog.streams.size()); + Assert.assertEquals("SEALED", streams.get(0).getCatalog().status); + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + Assert.assertFalse(Path.of(entry.manifest.file).isAbsolute()); + Assert.assertEquals(64, entry.manifest.sha256.length()); + Assert.assertEquals(64, entry.segments.get(0).sha256.length()); + } + } + + @Test + public void testRootManifestRejectsUnlistedFile() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + sealCatalog(root); + Files.writeString(root.resolve("unlisted.txt"), "not part of the backup"); + assertReadFails(root, false); + assertReadFails(root, true); + } + + @Test + public void testIncompleteOptInCannotBypassRootChecksum() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + sealCatalog(root); + final Path segment = onlySegment(root.resolve("data")); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.seek(LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.RECORD_HEADER_SIZE); + file.writeByte(0); + } + assertReadFails(root, false); + assertReadFails(root, true); + } + + @Test + public void testRootManifestRejectsTraversalAndDuplicateFiles() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + final LogicalBackupCatalog catalog = sealCatalog(root); + final String file = catalog.streams.get(0).manifest.file; + catalog.streams.get(0).manifest.file = "../outside/manifest.json"; + catalog.write(root); + assertReadFails(root, true); + catalog.streams.get(0).manifest.file = file; + catalog.streams.add(catalog.streams.get(0)); + catalog.write(root); + assertReadFails(root, true); + } + + @Test + public void testIncompleteStatusRequiresExplicitOptIn() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + final Path directory = root.resolve("data"); + try (final LogicalBackupWriter writer = writer(directory, manifest("data", "data"), 4096)) { + writer.recordSkippedEvent(1, "unsupported"); + } + final LogicalBackupCatalog catalog = sealCatalog(root); + Assert.assertEquals("INCOMPLETE", catalog.status); + Assert.assertEquals(1, catalog.skippedEventCount); + assertReadFails(root, false); + Assert.assertEquals(1, new LogicalBackupArchiveReader().read(root, true).size()); + } + + @Test + public void testAllowIncompleteStillChecksSealedCounters() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "data", "data", 4096); + final Path path = directory.resolve("manifest.json"); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(path), LogicalBackupManifest.class); + manifest.operationCounts.clear(); + Files.writeString(path, GSON.toJson(manifest)); + assertReadFails(directory, true); + } + + private static LogicalBackupCatalog sealCatalog(final Path root) throws IOException { + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = "backup"; + catalog.pipeName = "pipe"; + catalog.scope = Arrays.asList("config", "schema", "data"); + catalog.model = Arrays.asList("tree", "table"); + catalog.seal(root, new LogicalBackupArchiveReader().read(root, true)); + return catalog; + } + + private static void writeStream( + final Path directory, final String streamId, final String streamType, final long segmentSize) + throws Exception { + try (final LogicalBackupWriter writer = + writer(directory, manifest(streamId, streamType), segmentSize)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + if (segmentSize < 1000) { + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "metadata"); + } + } + } + + private static LogicalBackupManifest manifest(final String streamId, final String streamType) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = streamId; + manifest.streamType = streamType; + manifest.timestampPrecision = "ms"; + return manifest; + } + + private static LogicalBackupWriter writer( + final Path directory, final LogicalBackupManifest manifest, final long segmentSize) + throws IOException { + return new LogicalBackupWriter( + directory, + manifest, + segmentSize, + (int) Math.min(MAX_RECORD_BYTES, segmentSize), + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return request((short) 10, body); + } + + private static TPipeTransferReq request(final short type) { + return request(type, new byte[] {1}); + } + + private static TPipeTransferReq request(final short type, final byte[] body) { + return new TPipeTransferReq().setVersion((byte) 1).setType(type).setBody(ByteBuffer.wrap(body)); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + return files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .collect(java.util.stream.Collectors.toList()); + } + } + + private static void assertReadFails(final Path source, final boolean allowIncomplete) + throws IOException { + try { + new LogicalBackupArchiveReader().read(source, allowIncomplete); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java new file mode 100644 index 0000000000000..cdef5ccafd8c0 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class LogicalBackupSettingsTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testInvalidSettingsFailBeforeCreatingDirectory() throws Exception { + final Path root = temporaryFolder.getRoot().toPath().resolve("backup"); + for (final Map invalid : + Arrays.asList( + Map.of("sink.backup-id", "../outside"), + Map.of("sink.backup-id", "bad\\path"), + Map.of("sink.backup-id", "CON"), + Map.of("sink.segment-size-bytes", "1024"), + Map.of("sink.max-record-bytes", "268435457"), + Map.of("sink.fsync-policy", "batch", "sink.fsync-batch-operations", "0"), + Map.of("sink.fsync-period-ms", "1000"), + Map.of("sink.include-heartbeat", "maybe"), + Map.of("sink.on-unsupported-event", "ignore"))) { + final Map values = new HashMap<>(invalid); + values.put("sink.dir", root.toString()); + try { + new LogicalBackupSettings(new PipeParameters(values), "backup"); + Assert.fail(invalid.toString()); + } catch (final IOException expected) { + Assert.assertFalse(Files.exists(root)); + } + } + } + + @Test + public void testAllStreamsUseTheSameWriterSettingsAndAppendIdentity() throws Exception { + final Path root = temporaryFolder.getRoot().toPath().resolve("backup"); + final Map values = new HashMap<>(); + values.put("sink.dir", root.toString()); + values.put("sink.resume", "append"); + values.put("sink.backup-id", "backup-id"); + values.put("sink.segment-size-bytes", "8192"); + values.put("sink.max-record-bytes", "1024"); + values.put("sink.fsync-policy", "periodic"); + values.put("sink.fsync-period-ms", "12"); + values.put("sink.capture-options", "{\"scope\":\"config,schema,data\"}"); + final LogicalBackupSettings settings = + new LogicalBackupSettings(new PipeParameters(values), "backup"); + for (final String type : Arrays.asList("config", "schema", "data")) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup-id"; + manifest.streamId = type; + manifest.streamType = type; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = settings.open(type, 1, manifest)) { + Assert.assertEquals(8192, writer.getManifest().segmentSizeBytes); + Assert.assertEquals(1024, writer.getManifest().maxRecordBytes); + Assert.assertEquals("periodic", writer.getManifest().fsyncPolicy); + } + values.put("sink.capture-options", "{\"scope\":\"data\"}"); + try { + new LogicalBackupSettings(new PipeParameters(values), "backup").open(type, 1, manifest); + Assert.fail(); + } catch (final IOException expected) { + // A different scope cannot silently append to an existing stream. + } + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java new file mode 100644 index 0000000000000..b4b97c233fb8c --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -0,0 +1,611 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.AbstractList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupWriterTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testRoundTripAndDuplicateEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + final TPipeTransferReq request = request((byte) 1, (short) 2, new byte[] {3, 4, 5}); + + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + } + + final LogicalBackupSegmentReader.ScanResult result = scanOnlySegment(directory); + Assert.assertTrue(result.isSealed()); + Assert.assertEquals(4, result.getRecords().size()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_BEGIN, result.getRecords().get(0).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.PIPE_REQUEST, result.getRecords().get(1).getRecordType()); + Assert.assertEquals(request, result.getRecords().get(1).toTPipeTransferReq()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_COMMIT, result.getRecords().get(2).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.STREAM_END, result.getRecords().get(3).getRecordType()); + Assert.assertEquals(0, result.getFooter().getFirstSequence()); + Assert.assertEquals(3, result.getFooter().getLastSequence()); + } + + @Test + public void testDuplicateEventWithDifferentDigestFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(eventId, 1, Collections.singletonList(request(1)), ""); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(request(2)), ""); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(eventId.toString())); + } + } + } + + @Test + public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID firstEventId = UUID.randomUUID(); + final TPipeTransferReq firstRequest = replayableRequest(1); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + Assert.assertEquals( + 4, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(replayableRequest(2)), "second")); + } + + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(2, streams.get(0).getEventGroups().size()); + } + + @Test + public void testOnlineRollbackAfterPartialEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final List requests = + new AbstractList() { + private int accesses; + + @Override + public TPipeTransferReq get(final int index) { + if (++accesses == 6) { + throw new IllegalStateException(); + } + return request(index + 1); + } + + @Override + public int size() { + return 2; + } + }; + try { + writer.writeEvent(UUID.randomUUID(), 1, requests, ""); + Assert.fail(); + } catch (final IllegalStateException expected) { + // Expected. + } + Assert.assertEquals( + 0, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(3)), "after-rollback")); + } + + final List records = scanOnlySegment(directory).getRecords(); + Assert.assertEquals(4, records.size()); + Assert.assertEquals(0, records.get(0).getSequence()); + Assert.assertEquals("after-rollback", records.get(0).getMetadata()); + } + + @Test + public void testRecoverIncompleteEventAndRebuildCounts() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + final long beginAndRequestLength = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + frameSize(0, 0) + frameSize(0, 1); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(beginAndRequestLength); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_BEGIN")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("PIPE_REQUEST")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_COMMIT")); + } + } + + @Test + public void testRecoverIncompleteFrame() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 2); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(2, writer.getManifest().lastSequence); + } + } + + @Test + public void testRecoverPartialFooter() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE + 8); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals( + 4, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES) + .scan(segments(directory).get(segments(directory).size() - 1), false) + .isSealed()); + } + + @Test + public void testRecoverCorruptFooterWhenManifestStillMarksSegmentActive() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long lastByteOffset = file.length() - 1; + file.seek(lastByteOffset); + final int lastByte = file.readUnsignedByte(); + file.seek(lastByteOffset); + file.writeByte(lastByte ^ 1); + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + manifest.status = "WRITING"; + manifest.closedAt = null; + manifest.segments.get(0).status = "ACTIVE"; + manifest.segments.get(0).sizeBytes -= LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + manifest.segments.get(0).sha256 = null; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + } + Assert.assertTrue(scanOnlySegment(directory).isSealed()); + } + + @Test + public void testChecksumDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long payloadOffset = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + + frameSize("metadata".length(), 0) + + LogicalBackupFormat.RECORD_HEADER_SIZE + + "metadata".length(); + file.seek(payloadOffset); + file.writeByte(file.readByte() ^ 1); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("CRC")); + } + } + + @Test + public void testFooterSequenceDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long footerOffset = file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + final byte[] footer = new byte[LogicalBackupFormat.SEGMENT_FOOTER_SIZE]; + file.seek(footerOffset); + file.readFully(footer); + ByteBuffer.wrap(footer).order(ByteOrder.BIG_ENDIAN).putLong(Integer.BYTES + Long.BYTES, 99); + ByteBuffer.wrap(footer) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + footer, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES)); + file.seek(footerOffset); + file.write(footer); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("metadata")); + } + } + + @Test + public void testSegmentRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + } + final List segments = segments(directory); + Assert.assertEquals(2, segments.size()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(0), false).isSealed()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed()); + } + + @Test + public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest(); + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, manifest, 500, 500, LogicalBackupWriter.FsyncPolicy.NONE, 1, 1, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + Assert.assertEquals(2, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testControlRecordRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 258, false)) { + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first"); + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second"); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals(1, writer.getManifest().lastSequence); + } + } + + @Test + public void testDirectoryLock() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("locked")); + } + } + } + + @Test + public void testFailIfDirectoryExists() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath(); + try { + writer(directory, 4096, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(directory.toString())); + } + } + + @Test + public void testAppendRejectsConfigurationMismatch() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + try { + writer(directory, 8192, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segmentSizeBytes")); + } + } + + @Test + public void testAppendRejectsParentSegmentPath() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + final Path directory = root.resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + final String originalSegment = manifest.segments.get(0).file; + Files.copy(directory.resolve(originalSegment), root.resolve("outside.pwal")); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segment path")); + } + + manifest.segments.get(0).file = originalSegment; + Files.writeString(manifestPath, GSON.toJson(manifest)); + try (final LogicalBackupWriter ignored = writer(directory, 4096, true)) { + // A failed append must release the directory lock. + } + } + + @Test + public void testAppendAdoptsOrphanHeaderOnlySegmentAndClearsClosedState() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + Files.write(orphan, segmentHeader(1, 123)); + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals("WRITING", writer.getManifest().status); + Assert.assertNull(writer.getManifest().closedAt); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals( + 1, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + + final LogicalBackupSegmentReader.ScanResult scan = + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(orphan, false); + Assert.assertTrue(scan.isSealed()); + Assert.assertEquals(123, scan.getCreatedAt()); + } + + @Test + public void testAppendRejectsNonEmptyOrphanSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + final byte[] invalidOrphan = + Arrays.copyOf(segmentHeader(1, 123), LogicalBackupFormat.SEGMENT_HEADER_SIZE + 1); + Files.write(orphan, invalidOrphan); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + // The extra byte is an incomplete record tail and must never be adopted. + } + } + + @Test + public void testAppendRejectsUnexpectedUnlistedSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path unexpected = directory.resolve(segmentFileName(99)); + Files.write(unexpected, segmentHeader(99, 123)); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("Unlisted")); + } + } + + @Test + public void testPeriodicFsyncAdvancesOnHeartbeat() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest, + 4096, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.PERIODIC, + 1000, + 100, + false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + Assert.assertEquals(-1, writer.getManifest().lastDurableSequence); + Thread.sleep(150); + writer.heartbeat(); + Assert.assertEquals( + writer.getManifest().lastSequence, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testMaxRecordBytesHasHardLimit() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + try { + new LogicalBackupWriter( + directory, + manifest, + 4096, + LogicalBackupFormat.MAX_RECORD_BYTES + 1, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + private static LogicalBackupWriter writer( + final Path directory, final long segmentSize, final boolean append) throws IOException { + return new LogicalBackupWriter( + directory, + manifest(), + segmentSize, + (int) Math.min(MAX_RECORD_BYTES, segmentSize), + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + append); + } + + private static LogicalBackupManifest manifest() { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; + return manifest; + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return request((byte) 1, (short) 2, body); + } + + private static TPipeTransferReq replayableRequest(final int value) { + return request((byte) 1, (short) 10, new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte version, final short type, final byte[] body) { + return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body)); + } + + private static LogicalBackupSegmentReader.ScanResult scanOnlySegment(final Path directory) + throws IOException { + return new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(onlySegment(directory), false); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + final Path[] segments = + files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .toArray(Path[]::new); + return Arrays.asList(segments); + } + } + + private static long frameSize(final int metadataBytes, final int payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String segmentFileName(final long segmentId) { + return String.format(java.util.Locale.ROOT, "segment-%020d.pwal", segmentId); + } + + private static byte[] segmentHeader(final long segmentId, final long createdAt) { + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(createdAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES)); + return bytes; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java index 3bfbd6dcefddb..bcfce467d3ff8 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java @@ -28,8 +28,10 @@ import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.PipeSnapshotEvent; import org.apache.iotdb.commons.pipe.event.PipeWritePlanEvent; +import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; import org.junit.Assert; import org.junit.Test; @@ -39,10 +41,62 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedList; import java.util.Optional; public class IoTDBNonDataRegionSourceTest { + @Test + public void testLogicalBackupParsesMetadataInsteadOfTransferringSnapshotFiles() throws Exception { + final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class); + final TestNonDataRegionSource source = + new TestNonDataRegionSource(queue, new PipeTaskMeta(new MetaProgressIndex(10L), 0)); + final java.lang.reflect.Field finite = + IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot"); + finite.setAccessible(true); + finite.set(source, true); + final java.lang.reflect.Field snapshots = + IoTDBNonDataRegionSource.class.getDeclaredField("historicalEvents"); + snapshots.setAccessible(true); + snapshots.set( + source, + new LinkedList<>( + Collections.singletonList( + Mockito.mock(PipeSnapshotEvent.class, Mockito.RETURNS_SELF)))); + source.skipSnapshotPrivilegeCheck = true; + source.hasBeenStarted.set(true); + Assert.assertTrue(source.supply() instanceof ProgressReportEvent); + Assert.assertTrue(source.snapshotParsed); + Mockito.verifyZeroInteractions(queue); + } + + @Test + public void testFiniteSnapshotReportsOrderedCompletionAndNeverReadsRealtime() throws Exception { + final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class); + final PipeTaskMeta task = new PipeTaskMeta(new MetaProgressIndex(10L), 0); + final TestNonDataRegionSource source = new TestNonDataRegionSource(queue, task); + final java.lang.reflect.Field finite = + IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot"); + finite.setAccessible(true); + finite.set(source, true); + source.hasBeenStarted.set(true); + final EnrichedEvent completion = source.supply(); + Assert.assertTrue(completion instanceof ProgressReportEvent); + Assert.assertEquals( + new MetaProgressIndex(IoTDBNonDataRegionSource.LOGICAL_BACKUP_COMPLETED_INDEX), + completion.getProgressIndex()); + Assert.assertNull(source.supply()); + task.updateProgressIndex(completion.getProgressIndex()); + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Mockito.verifyZeroInteractions(queue); + + final TestNonDataRegionSource restarted = new TestNonDataRegionSource(queue, task); + finite.set(restarted, true); + restarted.start(); + Assert.assertNull(restarted.supply()); + Mockito.verifyZeroInteractions(queue); + } + @Test public void testStartWithStateWrappedHybridProgressIndex() throws Exception { final AbstractPipeListeningQueue listeningQueue = @@ -94,6 +148,8 @@ public void testGetUnTransferredEventCountWithHybridProgressIndexWithoutMetaInde private static final class TestNonDataRegionSource extends IoTDBNonDataRegionSource { private final AbstractPipeListeningQueue listeningQueue; + private boolean skipSnapshotPrivilegeCheck; + private boolean snapshotParsed; private TestNonDataRegionSource( final AbstractPipeListeningQueue listeningQueue, final PipeTaskMeta pipeTaskMeta) { @@ -123,13 +179,13 @@ protected long getMaxBlockingTimeMs() { @Override protected boolean canSkipSnapshotPrivilegeCheck(final PipeSnapshotEvent event) { - return false; + return skipSnapshotPrivilegeCheck; } @Override protected void initSnapshotGenerator(final PipeSnapshotEvent event) throws IOException, IllegalPathException { - // Do nothing + snapshotParsed = true; } @Override diff --git a/scripts/tools/export-logical-backup.sh b/scripts/tools/export-logical-backup.sh new file mode 100644 index 0000000000000..fcc072bb5c1ea --- /dev/null +++ b/scripts/tools/export-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/logical-backup.sh" export "$@" diff --git a/scripts/tools/import-logical-backup.sh b/scripts/tools/import-logical-backup.sh new file mode 100644 index 0000000000000..ebd38227a2896 --- /dev/null +++ b/scripts/tools/import-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/logical-backup.sh" import "$@" diff --git a/scripts/tools/logical-backup.sh b/scripts/tools/logical-backup.sh new file mode 100644 index 0000000000000..a0f6a3529f746 --- /dev/null +++ b/scripts/tools/logical-backup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [ -n "${IOTDB_INCLUDE}" ] && [ -r "${IOTDB_INCLUDE}" ]; then + . "${IOTDB_INCLUDE}" +fi + +if [ -z "${IOTDB_HOME}" ]; then + IOTDB_HOME="$(cd "$(dirname "$0")/.."; pwd)" + export IOTDB_HOME +fi + +if [ -n "${JAVA_HOME}" ] && [ -x "${JAVA_HOME}/bin/java" ]; then + JAVA="${JAVA_HOME}/bin/java" +else + JAVA=java +fi + +CLASSPATH="" +for jar in "${IOTDB_HOME}"/lib/*.jar; do + CLASSPATH="${CLASSPATH}:${jar}" +done + +exec "${JAVA}" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 \ + -DIOTDB_HOME="${IOTDB_HOME}" -cp "${CLASSPATH}" \ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool "$@" diff --git a/scripts/tools/windows/export-logical-backup.bat b/scripts/tools/windows/export-logical-backup.bat new file mode 100644 index 0000000000000..d79ce5a43d986 --- /dev/null +++ b/scripts/tools/windows/export-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0logical-backup.bat" export %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/import-logical-backup.bat b/scripts/tools/windows/import-logical-backup.bat new file mode 100644 index 0000000000000..bfe85ba3943fe --- /dev/null +++ b/scripts/tools/windows/import-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0logical-backup.bat" import %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/logical-backup.bat b/scripts/tools/windows/logical-backup.bat new file mode 100644 index 0000000000000..14e3180e46994 --- /dev/null +++ b/scripts/tools/windows/logical-backup.bat @@ -0,0 +1,40 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0..\.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED JAVA_HOME goto :no_java +set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +"%JAVA_HOME%\bin\java" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 ^ + -DIOTDB_HOME="%IOTDB_HOME%" -cp %CLASSPATH% ^ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool %* +set ret_code=%ERRORLEVEL% +goto :finally + +:no_java +echo JAVA_HOME environment variable must be set! +set ret_code=1 + +:finally +endlocal & exit /B %ret_code%