diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster index a4987f9ca88..95253ece91d 100755 --- a/assemble/bin/accumulo-cluster +++ b/assemble/bin/accumulo-cluster @@ -308,7 +308,7 @@ function parse_config() { RG_FILE="$AC_TMP_DIR/ZooInfoViewer.out" debug "Printing resource groups to $RG_FILE" - "$accumulo_cmd" zoo-info-viewer --print-resource-groups >"$RG_FILE" || (echo "Error getting resource groups. Did you init?" && exit 1) + "$accumulo_cmd" other zoo-info-viewer --print-resource-groups >"$RG_FILE" || (echo "Error getting resource groups. Did you init?" && exit 1) read -r -a all_resource_groups <<<"$(grep -F 'Resource Groups: ' "$RG_FILE" | cut -c18-)" debug "All resource groups: ${all_resource_groups[*]}" @@ -466,7 +466,7 @@ function control_services() { fi done # Try to cleanly stop the TabletServers and Manager - if ! "$accumulo_cmd" admin stopAll; then + if ! "$accumulo_cmd" admin stop-all; then echo "Invalid password or unable to connect to the manager" echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)" sleep 10 @@ -720,7 +720,7 @@ function prune() { exit 1 fi local service_json="$AC_TMP_DIR/accumulo-service.json" - "$accumulo_cmd" admin serviceStatus --json >"$service_json" 2>/dev/null || (echo "Error calling 'admin serviceStatus'" && exit 1) + "$accumulo_cmd" admin service-status --json >"$service_json" 2>/dev/null || (echo "Error calling 'admin serviceStatus'" && exit 1) local var_name local hosts diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java index 55834c48663..8e8b852212d 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java @@ -36,7 +36,7 @@ import org.apache.accumulo.core.manager.thrift.ManagerGoalState; import org.apache.accumulo.manager.state.SetGoalState; import org.apache.accumulo.minicluster.ServerType; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopAll; import org.apache.hadoop.util.Shell.ExitCodeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,7 +133,7 @@ private String sanitize(String msg) { @Override public void adminStopAll() throws IOException { String manager = getHosts(MANAGER_HOSTS_FILE).get(0); - String[] cmd = {serverCmdPrefix, accumuloPath, Admin.class.getName(), "stopAll"}; + String[] cmd = {serverCmdPrefix, accumuloPath, StopAll.class.getName()}; // Directly invoke the RemoteShell Entry pair = exec(manager, cmd); if (pair.getKey() != 0) { diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java index fc6762d7306..fab6e77b9cb 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java @@ -41,8 +41,8 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.server.conf.store.ResourceGroupPropKey; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.ZooZap; +import org.apache.accumulo.server.util.adminCommand.StopAll; import org.apache.commons.lang3.ArrayUtils; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -107,7 +107,7 @@ public Entry execWithStdout(Class clz, String[] args) throws @Override public void adminStopAll() throws IOException { - Process p = cluster.exec(Admin.class, "stopAll").getProcess(); + Process p = cluster.exec(StopAll.class).getProcess(); try { p.waitFor(); } catch (InterruptedException e) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/CheckAccumuloProperties.java b/server/base/src/main/java/org/apache/accumulo/server/conf/CheckAccumuloProperties.java index 9f4769f5949..ed8fdb06b5d 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/CheckAccumuloProperties.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/CheckAccumuloProperties.java @@ -24,7 +24,7 @@ import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.server.ServerDirs; import org.apache.accumulo.server.fs.VolumeManagerImpl; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.hadoop.conf.Configuration; @@ -46,7 +46,7 @@ public String description() { return "Checks the provided Accumulo configuration file for errors. " + "This only checks the contents of the file and not any running Accumulo system, " + "so it can be used prior to init, but only performs a subset of the checks done by " - + "'admin check run " + Admin.CheckCommand.Check.SERVER_CONFIG + "'"; + + "'admin check run " + Check.SERVER_CONFIG + "'"; } @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "intentional user-provided path") diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java deleted file mode 100644 index a2bce872c58..00000000000 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ /dev/null @@ -1,1500 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.util; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Formatter; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.NamespaceNotFoundException; -import org.apache.accumulo.core.client.ResourceGroupNotFoundException; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.admin.InstanceOperations; -import org.apache.accumulo.core.client.admin.servers.ServerId; -import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.ResourceGroupId; -import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.fate.AdminUtil; -import org.apache.accumulo.core.fate.FateId; -import org.apache.accumulo.core.fate.FateInstanceType; -import org.apache.accumulo.core.fate.FateStore; -import org.apache.accumulo.core.fate.ReadOnlyFateStore; -import org.apache.accumulo.core.fate.user.UserFateStore; -import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; -import org.apache.accumulo.core.fate.zookeeper.ZooUtil; -import org.apache.accumulo.core.lock.ServiceLock; -import org.apache.accumulo.core.lock.ServiceLockData; -import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector; -import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate; -import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; -import org.apache.accumulo.core.manager.thrift.FateService; -import org.apache.accumulo.core.manager.thrift.TFateId; -import org.apache.accumulo.core.metadata.SystemTables; -import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.process.thrift.ServerProcessService; -import org.apache.accumulo.core.rpc.ThriftUtil; -import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.NamespacePermission; -import org.apache.accumulo.core.security.SystemPermission; -import org.apache.accumulo.core.security.TablePermission; -import org.apache.accumulo.core.trace.TraceUtil; -import org.apache.accumulo.core.util.AddressUtil; -import org.apache.accumulo.core.util.Halt; -import org.apache.accumulo.core.zookeeper.ZooCache; -import org.apache.accumulo.core.zookeeper.ZooSession; -import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.security.SecurityUtil; -import org.apache.accumulo.server.util.checkCommand.CheckRunner; -import org.apache.accumulo.server.util.checkCommand.MetadataTableCheckRunner; -import org.apache.accumulo.server.util.checkCommand.RootMetadataCheckRunner; -import org.apache.accumulo.server.util.checkCommand.RootTableCheckRunner; -import org.apache.accumulo.server.util.checkCommand.ServerConfigCheckRunner; -import org.apache.accumulo.server.util.checkCommand.SystemConfigCheckRunner; -import org.apache.accumulo.server.util.checkCommand.SystemFilesCheckRunner; -import org.apache.accumulo.server.util.checkCommand.TableLocksCheckRunner; -import org.apache.accumulo.server.util.checkCommand.UserFilesCheckRunner; -import org.apache.accumulo.server.util.fateCommand.FateSummaryReport; -import org.apache.accumulo.start.spi.KeywordExecutable; -import org.apache.thrift.TException; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.MissingCommandException; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.auto.service.AutoService; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Lists; -import com.google.common.net.HostAndPort; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -@AutoService(KeywordExecutable.class) -public class Admin implements KeywordExecutable { - private static final Logger log = LoggerFactory.getLogger(Admin.class); - private final CountDownLatch lockAcquiredLatch = new CountDownLatch(1); - - @Deprecated(since = "4.0.0") - private static final String LOCKS_COMMAND = "locks"; - - private static class SubCommandOpts { - @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true) - public boolean help = false; - } - - @Parameters(commandNames = "stop", - commandDescription = "Stop the servers at the given addresses allowing them to complete current task but not start new task. Hostnames only are no longer supported; you must use . To Stop all services on a host, use 'accumulo admin serviceStatus' to list all hosts and then pass them to this command.") - static class StopCommand extends SubCommandOpts { - @Parameter(names = {"-f", "--force"}, - description = "force the given server to stop immediately by removing its lock. Does not wait for any task the server is currently working.") - boolean force = false; - @Parameter(description = " { ... }") - List args = new ArrayList<>(); - } - - @Parameters(commandNames = "ping", - commandDescription = "Ping tablet servers. If no arguments, pings all.") - static class PingCommand extends SubCommandOpts { - @Parameter(description = "{ ... }") - List args = new ArrayList<>(); - } - - @Parameters(commandNames = "check", - commandDescription = "Performs checks for problems in Accumulo.") - public static class CheckCommand extends SubCommandOpts { - @Parameter(names = "list", - description = "Lists the different checks that can be run, the description of each check, and the other check(s) each check depends on.") - boolean list; - - @Parameter(names = "run", - description = "Runs the provided check(s) (explicit list or regex pattern specified following '-p'), beginning with their dependencies, or all checks if none are provided.") - boolean run; - - @Parameter(names = {"--name_pattern", "-p"}, - description = "Runs all checks that match the provided regex pattern.") - String pattern; - - @Parameter(description = "[...]") - List checks; - - @Parameter(names = "--fixFiles", description = "Removes dangling file pointers. Used by the " - + "USER_FILES and SYSTEM_FILES checks.") - boolean fixFiles = false; - - /** - * This should be used to get the check runner instead of {@link Check#getCheckRunner()}. This - * exists so that its functionality can be changed for testing. - * - * @return the interface for running a check - */ - public CheckRunner getCheckRunner(Check check) { - return check.getCheckRunner(); - } - - public enum Check { - // Caution should be taken when changing or adding any new checks: order is important - SYSTEM_CONFIG(SystemConfigCheckRunner::new, "Validate the system config stored in ZooKeeper", - Collections.emptyList()), - SERVER_CONFIG(ServerConfigCheckRunner::new, "Validate the server configuration", - Collections.singletonList(SYSTEM_CONFIG)), - TABLE_LOCKS(TableLocksCheckRunner::new, - "Ensures that table and namespace locks are valid and are associated with a FATE op", - Collections.singletonList(SYSTEM_CONFIG)), - ROOT_METADATA(RootMetadataCheckRunner::new, - "Checks integrity of the root tablet metadata stored in ZooKeeper", - Collections.singletonList(SYSTEM_CONFIG)), - ROOT_TABLE(RootTableCheckRunner::new, - "Scans all the tablet metadata stored in the root table and checks integrity", - Collections.singletonList(ROOT_METADATA)), - METADATA_TABLE(MetadataTableCheckRunner::new, - "Scans all the tablet metadata stored in the metadata table and checks integrity", - Collections.singletonList(ROOT_TABLE)), - SYSTEM_FILES(SystemFilesCheckRunner::new, - "Checks that files in system tablet metadata exist in DFS", - Collections.singletonList(ROOT_TABLE)), - USER_FILES(UserFilesCheckRunner::new, - "Checks that files in user tablet metadata exist in DFS", - Collections.singletonList(METADATA_TABLE)); - - private final Supplier checkRunner; - private final String description; - private final List dependencies; - - Check(Supplier checkRunner, String description, List dependencies) { - this.checkRunner = Objects.requireNonNull(checkRunner); - this.description = Objects.requireNonNull(description); - this.dependencies = Objects.requireNonNull(dependencies); - } - - /** - * This should not be called directly; use {@link CheckCommand#getCheckRunner(Check)} instead - * - * @return the interface for running a check - */ - public CheckRunner getCheckRunner() { - return checkRunner.get(); - } - - /** - * @return the description of the check - */ - public String getDescription() { - return description; - } - - /** - * @return the list of other checks the check depends on - */ - public List getDependencies() { - return dependencies; - } - } - - public enum CheckStatus { - OK, FAILED, SKIPPED_DEPENDENCY_FAILED, FILTERED_OUT; - } - } - - @Parameters(commandNames = "stopManager", commandDescription = "stop the manager") - static class StopManagerCommand extends SubCommandOpts {} - - @Parameters(commandNames = "stopAll", - commandDescription = "stop all tablet servers and the manager") - static class StopAllCommand extends SubCommandOpts {} - - @Parameters(commandNames = "listInstances", - commandDescription = "list Accumulo instances in zookeeper") - static class ListInstancesCommand extends SubCommandOpts { - @Parameter(names = "--print-errors", description = "display errors while listing instances") - boolean printErrors = false; - @Parameter(names = "--print-all", - description = "print information for all instances, not just those with names") - boolean printAll = false; - } - - @Parameters(commandNames = "volumes", commandDescription = "Accumulo volume utility") - static class VolumesCommand extends SubCommandOpts { - @Parameter(names = {"-l", "--list"}, description = "list volumes currently in use") - boolean printErrors = false; - } - - @Parameters(commandNames = "dumpConfig", - commandDescription = "print out non-default configuration settings") - static class DumpConfigCommand extends SubCommandOpts { - @Parameter(names = {"-a", "--all"}, - description = "print the system and all table configurations") - boolean allConfiguration = false; - @Parameter(names = {"-d", "--directory"}, description = "directory to place config files") - String directory = null; - @Parameter(names = {"-s", "--system"}, description = "print the system configuration") - boolean systemConfiguration = false; - @Parameter(names = {"-rg", "--resourceGroups"}, - description = "print the resource group configuration") - boolean resourceGroupConfiguration = false; - @Parameter(names = {"-n", "--namespaces"}, description = "print the namespace configuration") - boolean namespaceConfiguration = false; - @Parameter(names = {"-t", "--tables"}, description = "print per-table configuration") - List tables = new ArrayList<>(); - @Parameter(names = {"-u", "--users"}, - description = "print users and their authorizations and permissions") - boolean users = false; - } - - @Parameters(commandNames = "verifyTabletAssigns", - commandDescription = "Verify all Tablets are assigned to tablet servers") - static class VerifyTabletAssignmentsCommand extends SubCommandOpts { - @Parameter(names = {"-v", "--verbose"}, - description = "verbose mode (prints locations of tablets)") - boolean verbose = false; - } - - /** - * @since 2.1.0 - */ - @Parameters(commandNames = "changeSecret", - commandDescription = "Changes the unique secret given to the instance that all servers must know.") - static class ChangeSecretCommand {} - - @Parameters(commandNames = "deleteZooInstance", - commandDescription = "Deletes specific instance name or id from zookeeper or cleans up all old instances.") - static class DeleteZooInstanceCommand extends SubCommandOpts { - @Parameter(names = {"-i", "--instance"}, description = "the instance name or id to delete") - String instance; - @Parameter(names = {"-c", "--clean"}, - description = "Cleans Zookeeper by deleting all old instances. This will not delete the instance pointed to by the local accumulo.properties file") - boolean clean = false; - @Parameter(names = {"--password"}, - description = "The system secret, if different than instance.secret in accumulo.properties", - password = true) - String auth; - } - - @Parameters(commandNames = "restoreZoo", - commandDescription = "Restore Zookeeper data from a file.") - static class RestoreZooCommand extends SubCommandOpts { - @Parameter(names = "--overwrite") - boolean overwrite = false; - - @Parameter(names = "--file") - String file; - } - - @Parameters(commandNames = "fate", - commandDescription = "Operations performed on the Manager FaTE system.") - static class FateOpsCommand extends SubCommandOpts { - @Parameter(description = "[...]") - List fateIdList = new ArrayList<>(); - - @Parameter(names = {"-c", "--cancel"}, - description = "... Cancel new or submitted FaTE transactions") - boolean cancel; - - @Parameter(names = {"-f", "--fail"}, - description = "... Transition FaTE transaction status to FAILED_IN_PROGRESS") - boolean fail; - - @Parameter(names = {"-d", "--delete"}, - description = "... Delete FaTE transaction and its associated table locks") - boolean delete; - - @Parameter(names = {"-p", "--print", "-print", "-l", "--list", "-list"}, - description = "[...] Print information about FaTE transactions. Print only the FateId's specified or print all transactions if empty. Use -s to only print those with certain states. Use -t to only print those with certain FateInstanceTypes.") - boolean print; - - @Parameter(names = "--summary", - description = "[...] Print a summary of FaTE transactions. Print only the FateId's specified or print all transactions if empty. Use -s to only print those with certain states. Use -t to only print those with certain FateInstanceTypes. Use -j to print the transactions in json.") - boolean summarize; - - @Parameter(names = {"-j", "--json"}, - description = "Print transactions in json. Only useful for --summary command.") - boolean printJson; - - @Parameter(names = {"-s", "--state"}, - description = "... Print transactions in the state(s) {NEW, IN_PROGRESS, FAILED_IN_PROGRESS, FAILED, SUCCESSFUL}") - List states = new ArrayList<>(); - - @Parameter(names = {"-t", "--type"}, - description = "... Print transactions of fate instance type(s) {USER, META}") - List instanceTypes = new ArrayList<>(); - } - - class AdminLockWatcher implements ServiceLock.AccumuloLockWatcher { - @Override - public void lostLock(ServiceLock.LockLossReason reason) { - String msg = "Admin lost lock: " + reason.toString(); - if (reason == ServiceLock.LockLossReason.LOCK_DELETED) { - Halt.halt(0, msg); - } else { - Halt.halt(1, msg); - } - } - - @Override - public void unableToMonitorLockNode(Exception e) { - String msg = "Admin unable to monitor lock: " + e.getMessage(); - log.warn(msg); - Halt.halt(1, msg); - } - - @Override - public void acquiredLock() { - lockAcquiredLatch.countDown(); - log.debug("Acquired ZooKeeper lock for Admin"); - } - - @Override - public void failedToAcquireLock(Exception e) { - log.warn("Failed to acquire ZooKeeper lock for Admin, msg: " + e.getMessage()); - } - } - - @Parameters(commandNames = "serviceStatus", commandDescription = "show service status") - public static class ServiceStatusCmdOpts extends SubCommandOpts { - @Parameter(names = "--json", description = "provide output in json format") - boolean json = false; - @Parameter(names = "--showHosts", - description = "provide a summary of service counts with host details") - boolean showHosts = false; - } - - public static void main(String[] args) { - new Admin().execute(args); - } - - @Override - public String keyword() { - return "admin"; - } - - @Override - public UsageGroup usageGroup() { - return UsageGroup.CORE; - } - - @Override - public String description() { - return "Executes administrative commands"; - } - - @SuppressFBWarnings(value = "DM_EXIT", justification = "System.exit okay for CLI tool") - @Override - public void execute(final String[] args) { - - ServerUtilOpts opts = new ServerUtilOpts(); - JCommander cl = new JCommander(opts); - cl.setProgramName("accumulo admin"); - - ServiceStatusCmdOpts serviceStatusCommandOpts = new ServiceStatusCmdOpts(); - cl.addCommand(serviceStatusCommandOpts); - - ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand(); - cl.addCommand(changeSecretCommand); - - CheckCommand checkCommand = new CheckCommand(); - cl.addCommand(checkCommand); - - DeleteZooInstanceCommand deleteZooInstOpts = new DeleteZooInstanceCommand(); - cl.addCommand(deleteZooInstOpts); - - DumpConfigCommand dumpConfigCommand = new DumpConfigCommand(); - cl.addCommand(dumpConfigCommand); - - FateOpsCommand fateOpsCommand = new FateOpsCommand(); - cl.addCommand(fateOpsCommand); - - ListInstancesCommand listInstancesOpts = new ListInstancesCommand(); - cl.addCommand(listInstancesOpts); - - PingCommand pingCommand = new PingCommand(); - cl.addCommand(pingCommand); - - RestoreZooCommand restoreZooOpts = new RestoreZooCommand(); - cl.addCommand(restoreZooOpts); - - StopCommand stopOpts = new StopCommand(); - cl.addCommand(stopOpts); - - StopAllCommand stopAllOpts = new StopAllCommand(); - cl.addCommand(stopAllOpts); - - StopManagerCommand stopManagerOpts = new StopManagerCommand(); - cl.addCommand(stopManagerOpts); - - VerifyTabletAssignmentsCommand verifyTabletAssignmentsOpts = - new VerifyTabletAssignmentsCommand(); - cl.addCommand(verifyTabletAssignmentsOpts); - - VolumesCommand volumesCommand = new VolumesCommand(); - cl.addCommand(volumesCommand); - - try { - cl.parse(args); - } catch (MissingCommandException e) { - // Process removed commands to provide alternate approach - boolean foundRemovedCommand = false; - for (String arg : args) { - switch (arg) { - case LOCKS_COMMAND: - foundRemovedCommand = true; - System.out.println("'locks' command has been removed. Use 'serviceStatus' command" - + " to list processes and 'stop -f' command to remove their locks."); - break; - default: - break; - } - } - if (foundRemovedCommand) { - return; - } else { - cl.usage(); - return; - } - } - - if (cl.getParsedCommand() == null) { - cl.usage(); - return; - } - - for (var command : cl.getCommands().entrySet()) { - var objects = command.getValue().getObjects(); - for (var obj : objects) { - if (obj instanceof SubCommandOpts && ((SubCommandOpts) obj).help) { - command.getValue().usage(); - return; - } - } - } - - try (ServerContext context = opts.getServerContext()) { - - AccumuloConfiguration conf = context.getConfiguration(); - // Login as the server on secure HDFS - if (conf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) { - SecurityUtil.serverLogin(conf); - } - - int rc = 0; - - if (cl.getParsedCommand().equals("listInstances")) { - ListInstances.listInstances(context.getZooKeepers(), listInstancesOpts.printAll, - listInstancesOpts.printErrors); - } else if (cl.getParsedCommand().equals("ping")) { - if (ping(context, pingCommand.args) != 0) { - rc = 4; - } - } else if (cl.getParsedCommand().equals("stop")) { - stopServers(context, stopOpts.args, stopOpts.force); - } else if (cl.getParsedCommand().equals("dumpConfig")) { - printConfig(context, dumpConfigCommand); - } else if (cl.getParsedCommand().equals("volumes")) { - ListVolumesUsed.listVolumes(context); - } else if (cl.getParsedCommand().equals("verifyTabletAssigns")) { - VerifyTabletAssignments.execute(context, verifyTabletAssignmentsOpts.verbose); - } else if (cl.getParsedCommand().equals("changeSecret")) { - ChangeSecret.execute(context, conf); - } else if (cl.getParsedCommand().equals("deleteZooInstance")) { - DeleteZooInstance.execute(context, deleteZooInstOpts.clean, deleteZooInstOpts.instance, - deleteZooInstOpts.auth); - } else if (cl.getParsedCommand().equals("restoreZoo")) { - RestoreZookeeper.execute(conf, restoreZooOpts.file, restoreZooOpts.overwrite); - } else if (cl.getParsedCommand().equals("fate")) { - executeFateOpsCommand(context, fateOpsCommand); - } else if (cl.getParsedCommand().equals("serviceStatus")) { - ServiceStatusCmd ssc = new ServiceStatusCmd(); - ssc.execute(context, serviceStatusCommandOpts.json, serviceStatusCommandOpts.showHosts); - } else if (cl.getParsedCommand().equals("check")) { - rc = executeCheckCommand(context, checkCommand, opts); - } else if (cl.getParsedCommand().equals("stopManager") - || cl.getParsedCommand().equals("stopAll")) { - boolean everything = cl.getParsedCommand().equals("stopAll"); - - if (everything) { - flushAll(context); - } - - stopServer(context, everything); - } else { - cl.usage(); - } - - if (rc != 0) { - System.exit(rc); - } - } catch (AccumuloException e) { - log.error("{}", e.getMessage(), e); - System.exit(1); - } catch (AccumuloSecurityException e) { - log.error("{}", e.getMessage(), e); - System.exit(2); - } catch (Exception e) { - log.error("{}", e.getMessage(), e); - System.exit(3); - } - } - - private static int ping(ClientContext context, List args) { - - InstanceOperations io = context.instanceOperations(); - - if (args.isEmpty()) { - io.getServers(ServerId.Type.TABLET_SERVER).forEach(t -> args.add(t.toHostPortString())); - } - - int unreachable = 0; - - for (String tserver : args) { - try { - io.ping(tserver); - System.out.println(tserver + " OK"); - } catch (AccumuloException ae) { - System.out.println(tserver + " FAILED (" + ae.getMessage() + ")"); - unreachable++; - } - } - - System.out.printf("\n%d of %d tablet servers unreachable\n\n", unreachable, args.size()); - return unreachable; - } - - /** - * Flushing during shutdown is a performance optimization, it's not required. This method will - * attempt to initiate flushes of all tables and give up if it takes too long. - */ - private static void flushAll(final ClientContext context) { - - final AtomicInteger flushesStarted = new AtomicInteger(0); - - Runnable flushTask = () -> { - try { - Set tables = context.tableOperations().list(); - for (String table : tables) { - if (table.equals(SystemTables.METADATA.tableName())) { - continue; - } - try { - context.tableOperations().flush(table, null, null, false); - flushesStarted.incrementAndGet(); - } catch (TableNotFoundException e) { - // ignore - } - } - } catch (Exception e) { - log.warn("Failed to initiate flush {}", e.getMessage()); - } - }; - - Thread flusher = new Thread(flushTask); - flusher.setDaemon(true); - flusher.start(); - - long start = System.currentTimeMillis(); - try { - flusher.join(3000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("Interrupted while waiting to join Flush thread", e); - } - - while (flusher.isAlive() && System.currentTimeMillis() - start < 15000) { - int flushCount = flushesStarted.get(); - try { - flusher.join(1000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("Interrupted while waiting to join Flush thread", e); - } - - if (flushCount == flushesStarted.get()) { - // no progress was made while waiting for join... maybe its stuck, stop waiting on it - break; - } - } - - flusher.interrupt(); - try { - flusher.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("Interrupted while waiting to join Flush thread", e); - } - } - - private static void stopServer(final ClientContext context, final boolean tabletServersToo) - throws AccumuloException, AccumuloSecurityException { - - ThriftClientTypes.MANAGER.executeVoid(context, - client -> client.shutdown(TraceUtil.traceInfo(), context.rpcCreds(), tabletServersToo)); - } - - private static void stopServers(final ServerContext context, List servers, - final boolean force) - throws AccumuloException, AccumuloSecurityException, InterruptedException, KeeperException { - List hostOnly = new ArrayList<>(); - Set hostAndPort = new TreeSet<>(); - - for (var server : servers) { - if (server.contains(":")) { - hostAndPort.add(server); - } else { - hostOnly.add(server); - } - } - - if (!hostOnly.isEmpty()) { - // The old impl of this command with the old behavior - log.warn("Stopping by hostname is no longer supported\n\n" - + "please use instead.\n" - + "To stop all services on host, run 'accumulo admin serviceStatus' to list all host:port values and filter for that host and pass those to 'accumulo admin stop'"); - stopTabletServer(context, hostOnly, force); - } - - if (!hostAndPort.isEmpty()) { - // New behavior for this command when ports are present, supports more than just tservers. Is - // also async. - if (force) { - var zoo = context.getZooSession().asReaderWriter(); - - AddressSelector addresses = AddressSelector.matching(hostAndPort::contains); - List pathsToRemove = new ArrayList<>(); - pathsToRemove.addAll( - context.getServerPaths().getCompactor(ResourceGroupPredicate.ANY, addresses, false)); - pathsToRemove.addAll( - context.getServerPaths().getScanServer(ResourceGroupPredicate.ANY, addresses, false)); - pathsToRemove.addAll( - context.getServerPaths().getTabletServer(ResourceGroupPredicate.ANY, addresses, false)); - ZooZap.filterSingleton(context, context.getServerPaths().getManager(false), addresses) - .ifPresent(pathsToRemove::add); - ZooZap.filterSingleton(context, context.getServerPaths().getGarbageCollector(false), - addresses).ifPresent(pathsToRemove::add); - ZooZap.filterSingleton(context, context.getServerPaths().getMonitor(false), addresses) - .ifPresent(pathsToRemove::add); - - for (var path : pathsToRemove) { - List children = zoo.getChildren(path.toString()); - for (String child : children) { - log.trace("removing lock {}", path + "/" + child); - zoo.recursiveDelete(path + "/" + child, ZooUtil.NodeMissingPolicy.SKIP); - } - } - } else { - for (var server : hostAndPort) { - signalGracefulShutdown(context, HostAndPort.fromString(server)); - } - } - } - } - - // Visible for tests - public static void signalGracefulShutdown(final ClientContext context, HostAndPort hp) { - Objects.requireNonNull(hp, "address not set"); - ServerProcessService.Client client = null; - try { - client = ThriftClientTypes.SERVER_PROCESS.getServerProcessConnection(context, log, - hp.getHost(), hp.getPort()); - if (client == null) { - log.warn("Failed to initiate shutdown for {}", hp); - return; - } - client.gracefulShutdown(context.rpcCreds()); - log.info("Initiated shutdown for {}", hp); - } catch (TException e) { - log.warn("Failed to initiate shutdown for {}", hp, e); - } finally { - if (client != null) { - ThriftUtil.returnClient(client, context); - } - } - } - - /** - * Stops tablet servers by hostname - * - * @param context The server context - * @param servers LIst of hostnames (without ports) - * @param force Whether to force stop - * @deprecated Use servers with host:port format instead. To stop all services on a host use - * service status command to liat all services, then stop them with host:port format. - */ - @Deprecated(since = "4.0.0") - private static void stopTabletServer(final ClientContext context, List servers, - final boolean force) throws AccumuloException, AccumuloSecurityException { - if (context.instanceOperations().getServers(ServerId.Type.MANAGER).isEmpty()) { - log.info("No managers running. Not attempting safe unload of tserver."); - return; - } - if (servers.isEmpty()) { - log.error("No tablet servers provided."); - return; - } - - final ZooCache zc = context.getZooCache(); - Set runningServers; - - for (String server : servers) { - runningServers = context.instanceOperations().getServers(ServerId.Type.TABLET_SERVER); - if (runningServers.size() == 1 && !force) { - log.info("Only 1 tablet server running. Not attempting shutdown of {}", server); - return; - } - for (int port : context.getConfiguration().getPort(Property.TSERV_CLIENTPORT)) { - HostAndPort address = AddressUtil.parseAddress(server, port); - final String finalServer = qualifyWithZooKeeperSessionId(context, zc, address.toString()); - log.info("Stopping server {}", finalServer); - ThriftClientTypes.MANAGER.executeVoid(context, client -> client - .shutdownTabletServer(TraceUtil.traceInfo(), context.rpcCreds(), finalServer, force)); - } - } - } - - /** - * Look up the TabletServers in ZooKeeper and try to find a sessionID for this server reference - * - * @param hostAndPort The host and port for a TabletServer - * @return The host and port with the session ID in square-brackets appended, or the original - * value. - */ - static String qualifyWithZooKeeperSessionId(ClientContext context, ZooCache zooCache, - String hostAndPort) { - var hpObj = HostAndPort.fromString(hostAndPort); - Set paths = context.getServerPaths() - .getTabletServer(ResourceGroupPredicate.ANY, AddressSelector.exact(hpObj), true); - if (paths.size() != 1) { - return hostAndPort; - } - long sessionId = ServiceLock.getSessionId(zooCache, paths.iterator().next()); - if (sessionId == 0) { - return hostAndPort; - } - return hostAndPort + "[" + Long.toHexString(sessionId) + "]"; - } - - private static final String ACCUMULO_SITE_BACKUP_FILE = "accumulo.properties.bak"; - private static final String NS_FILE_SUFFIX = "_ns.cfg"; - private static final String RG_FILE_SUFFIX = "_rg.cfg"; - private static final String USER_FILE_SUFFIX = "_user.cfg"; - private static final MessageFormat configFormat = new MessageFormat("config -t {0} -s {1}\n"); - private static final MessageFormat createNsFormat = new MessageFormat("createnamespace {0}\n"); - private static final MessageFormat createTableFormat = new MessageFormat("createtable {0}\n"); - private static final MessageFormat createUserFormat = new MessageFormat("createuser {0}\n"); - private static final MessageFormat createRGFormat = - new MessageFormat("createresourcegroup {0}\n"); - private static final MessageFormat nsConfigFormat = new MessageFormat("config -ns {0} -s {1}\n"); - private static final MessageFormat rgConfigFormat = new MessageFormat("config -rg {0} -s {1}\n"); - private static final MessageFormat sysPermFormat = - new MessageFormat("grant System.{0} -s -u {1}\n"); - private static final MessageFormat nsPermFormat = - new MessageFormat("grant Namespace.{0} -ns {1} -u {2}\n"); - private static final MessageFormat tablePermFormat = - new MessageFormat("grant Table.{0} -t {1} -u {2}\n"); - private static final MessageFormat userAuthsFormat = - new MessageFormat("setauths -u {0} -s {1}\n"); - - private DefaultConfiguration defaultConfig; - private Map siteConfig; - private Map systemConfig; - private List localUsers; - - public void printConfig(ClientContext context, DumpConfigCommand opts) throws Exception { - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "app is run in same security context as user providing the filename") - File outputDirectory = getOutputDirectory(opts.directory); - defaultConfig = DefaultConfiguration.getInstance(); - siteConfig = context.instanceOperations().getSiteConfiguration(); - systemConfig = context.instanceOperations().getSystemConfiguration(); - if (opts.allConfiguration || opts.users) { - localUsers = Lists.newArrayList(context.securityOperations().listLocalUsers()); - Collections.sort(localUsers); - } - - if (opts.allConfiguration) { - // print accumulo site - printSystemConfiguration(outputDirectory); - // print resource groups - for (ResourceGroupId group : context.resourceGroupOperations().list()) { - printResourceGroupConfiguration(context, group, outputDirectory); - } - // print namespaces - for (String namespace : context.namespaceOperations().list()) { - printNameSpaceConfiguration(context, namespace, outputDirectory); - } - // print tables - SortedSet tableNames = context.tableOperations().list(); - for (String tableName : tableNames) { - printTableConfiguration(context, tableName, outputDirectory); - } - // print users - for (String user : localUsers) { - printUserConfiguration(context, user, outputDirectory); - } - } else { - if (opts.systemConfiguration) { - printSystemConfiguration(outputDirectory); - } - if (opts.resourceGroupConfiguration) { - for (ResourceGroupId group : context.resourceGroupOperations().list()) { - printResourceGroupConfiguration(context, group, outputDirectory); - } - } - if (opts.namespaceConfiguration) { - for (String namespace : context.namespaceOperations().list()) { - printNameSpaceConfiguration(context, namespace, outputDirectory); - } - } - if (!opts.tables.isEmpty()) { - for (String tableName : opts.tables) { - printTableConfiguration(context, tableName, outputDirectory); - } - } - if (opts.users) { - for (String user : localUsers) { - printUserConfiguration(context, user, outputDirectory); - } - } - } - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "app is run in same security context as user providing the filename") - private static File getOutputDirectory(final String directory) { - if (directory == null) { - return null; - } - Path outputDirectory = Path.of(directory); - if (!Files.isDirectory(outputDirectory)) { - throw new IllegalArgumentException(directory + " does not exist on the local filesystem."); - } - if (!Files.isWritable(outputDirectory)) { - throw new IllegalArgumentException(directory + " is not writable"); - } - return outputDirectory.toFile(); - } - - private String getDefaultConfigValue(String key) { - if (key == null) { - return null; - } - - String defaultValue = null; - try { - Property p = Property.getPropertyByKey(key); - if (p == null) { - return defaultValue; - } - defaultValue = defaultConfig.get(p); - } catch (IllegalArgumentException e) { - // ignore - } - return defaultValue; - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "code runs in same security context as user who provided input") - private void printResourceGroupConfiguration(AccumuloClient accumuloClient, ResourceGroupId group, - File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException, - ResourceGroupNotFoundException { - Path rgScript = outputDirectory.toPath().resolve(group + RG_FILE_SUFFIX); - try (BufferedWriter nsWriter = Files.newBufferedWriter(rgScript)) { - nsWriter.write(createRGFormat.format(new String[] {group.canonical()})); - Map props = - ImmutableSortedMap.copyOf(accumuloClient.resourceGroupOperations().getProperties(group)); - for (Entry entry : props.entrySet()) { - String defaultValue = getDefaultConfigValue(entry.getKey()); - if (defaultValue == null || !defaultValue.equals(entry.getValue())) { - if (!entry.getValue().equals(siteConfig.get(entry.getKey())) - && !entry.getValue().equals(systemConfig.get(entry.getKey()))) { - nsWriter.write(rgConfigFormat - .format(new String[] {group.canonical(), entry.getKey() + "=" + entry.getValue()})); - } - } - } - } - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "code runs in same security context as user who provided input") - private void printNameSpaceConfiguration(AccumuloClient accumuloClient, String namespace, - File outputDirectory) - throws IOException, AccumuloException, AccumuloSecurityException, NamespaceNotFoundException { - Path namespaceScript = outputDirectory.toPath().resolve(namespace + NS_FILE_SUFFIX); - try (BufferedWriter nsWriter = Files.newBufferedWriter(namespaceScript)) { - nsWriter.write(createNsFormat.format(new String[] {namespace})); - Map props = ImmutableSortedMap - .copyOf(accumuloClient.namespaceOperations().getConfiguration(namespace)); - for (Entry entry : props.entrySet()) { - String defaultValue = getDefaultConfigValue(entry.getKey()); - if (defaultValue == null || !defaultValue.equals(entry.getValue())) { - if (!entry.getValue().equals(siteConfig.get(entry.getKey())) - && !entry.getValue().equals(systemConfig.get(entry.getKey()))) { - nsWriter.write(nsConfigFormat - .format(new String[] {namespace, entry.getKey() + "=" + entry.getValue()})); - } - } - } - } - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "code runs in same security context as user who provided input") - private static void printUserConfiguration(AccumuloClient accumuloClient, String user, - File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException { - Path userScript = outputDirectory.toPath().resolve(user + USER_FILE_SUFFIX); - try (BufferedWriter userWriter = Files.newBufferedWriter(userScript)) { - userWriter.write(createUserFormat.format(new String[] {user})); - Authorizations auths = accumuloClient.securityOperations().getUserAuthorizations(user); - userWriter.write(userAuthsFormat.format(new String[] {user, auths.toString()})); - for (SystemPermission sp : SystemPermission.values()) { - if (accumuloClient.securityOperations().hasSystemPermission(user, sp)) { - userWriter.write(sysPermFormat.format(new String[] {sp.name(), user})); - } - } - for (String namespace : accumuloClient.namespaceOperations().list()) { - for (NamespacePermission np : NamespacePermission.values()) { - if (accumuloClient.securityOperations().hasNamespacePermission(user, namespace, np)) { - userWriter.write(nsPermFormat.format(new String[] {np.name(), namespace, user})); - } - } - } - for (String tableName : accumuloClient.tableOperations().list()) { - for (TablePermission perm : TablePermission.values()) { - if (accumuloClient.securityOperations().hasTablePermission(user, tableName, perm)) { - userWriter.write(tablePermFormat.format(new String[] {perm.name(), tableName, user})); - } - } - } - } - } - - private void printSystemConfiguration(File outputDirectory) throws IOException { - TreeMap conf = new TreeMap<>(); - TreeMap site = new TreeMap<>(siteConfig); - for (Entry prop : site.entrySet()) { - String defaultValue = getDefaultConfigValue(prop.getKey()); - if (!prop.getValue().equals(defaultValue) && !systemConfig.containsKey(prop.getKey())) { - conf.put(prop.getKey(), prop.getValue()); - } - } - TreeMap system = new TreeMap<>(systemConfig); - for (Entry prop : system.entrySet()) { - String defaultValue = getDefaultConfigValue(prop.getKey()); - if (!prop.getValue().equals(defaultValue)) { - conf.put(prop.getKey(), prop.getValue()); - } - } - Path siteBackup = outputDirectory.toPath().resolve(ACCUMULO_SITE_BACKUP_FILE); - try (BufferedWriter fw = Files.newBufferedWriter(siteBackup)) { - for (Entry prop : conf.entrySet()) { - fw.write(prop.getKey() + "=" + prop.getValue() + "\n"); - } - } - } - - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "code runs in same security context as user who provided input") - private void printTableConfiguration(AccumuloClient accumuloClient, String tableName, - File outputDirectory) throws AccumuloException, TableNotFoundException, IOException { - Path tableBackup = outputDirectory.toPath().resolve(tableName + ".cfg"); - try (BufferedWriter writer = Files.newBufferedWriter(tableBackup)) { - writer.write(createTableFormat.format(new String[] {tableName})); - Map props = - ImmutableSortedMap.copyOf(accumuloClient.tableOperations().getConfiguration(tableName)); - for (Entry prop : props.entrySet()) { - if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) { - String defaultValue = getDefaultConfigValue(prop.getKey()); - if (defaultValue == null || !defaultValue.equals(prop.getValue())) { - if (!prop.getValue().equals(siteConfig.get(prop.getKey())) - && !prop.getValue().equals(systemConfig.get(prop.getKey()))) { - writer.write(configFormat - .format(new String[] {tableName, prop.getKey() + "=" + prop.getValue()})); - } - } - } - } - } - } - - // Fate Operations - private void executeFateOpsCommand(ServerContext context, FateOpsCommand fateOpsCommand) - throws AccumuloException, AccumuloSecurityException, InterruptedException, KeeperException, - NamespaceNotFoundException { - - validateFateUserInput(fateOpsCommand); - - AdminUtil admin = new AdminUtil<>(); - var zTableLocksPath = context.getServerPaths().createTableLocksPath(); - var zk = context.getZooSession(); - ServiceLock adminLock = null; - Map> readOnlyFateStores = null; - - try { - if (fateOpsCommand.cancel) { - cancelSubmittedFateTxs(context, fateOpsCommand.fateIdList); - } else if (fateOpsCommand.fail) { - adminLock = createAdminLock(context); - try (var fateStores = createFateStores(context, zk, adminLock)) { - for (String fateIdStr : fateOpsCommand.fateIdList) { - if (!admin.prepFail(fateStores.getStoresMap(), fateIdStr)) { - throw new AccumuloException("Could not fail transaction: " + fateIdStr); - } - } - } - } else if (fateOpsCommand.delete) { - adminLock = createAdminLock(context); - try (var fateStores = createFateStores(context, zk, adminLock)) { - for (String fateIdStr : fateOpsCommand.fateIdList) { - if (!admin.prepDelete(fateStores.getStoresMap(), fateIdStr)) { - throw new AccumuloException("Could not delete transaction: " + fateIdStr); - } - admin.deleteLocks(zk, zTableLocksPath, fateIdStr); - } - } - } - - if (fateOpsCommand.print) { - final Set fateIdFilter = new TreeSet<>(); - fateOpsCommand.fateIdList.forEach(fateIdStr -> fateIdFilter.add(FateId.from(fateIdStr))); - EnumSet statusFilter = - getCmdLineStatusFilters(fateOpsCommand.states); - EnumSet typesFilter = - getCmdLineInstanceTypeFilters(fateOpsCommand.instanceTypes); - readOnlyFateStores = createReadOnlyFateStores(context, zk); - admin.print(readOnlyFateStores, zk, zTableLocksPath, new Formatter(System.out), - fateIdFilter, statusFilter, typesFilter); - // print line break at the end - System.out.println(); - } - - if (fateOpsCommand.summarize) { - if (readOnlyFateStores == null) { - readOnlyFateStores = createReadOnlyFateStores(context, zk); - } - summarizeFateTx(context, fateOpsCommand, admin, readOnlyFateStores, zTableLocksPath); - } - } finally { - if (adminLock != null) { - adminLock.unlock(); - } - } - } - - private FateStores createFateStores(ServerContext context, ZooSession zk, ServiceLock adminLock) - throws InterruptedException, KeeperException { - var lockId = adminLock.getLockID(); - MetaFateStore mfs = new MetaFateStore<>(zk, lockId, null); - UserFateStore ufs = - new UserFateStore<>(context, SystemTables.FATE.tableName(), lockId, null); - return new FateStores(FateInstanceType.META, mfs, FateInstanceType.USER, ufs); - } - - private Map> createReadOnlyFateStores( - ServerContext context, ZooSession zk) throws InterruptedException, KeeperException { - ReadOnlyFateStore readOnlyMFS = new MetaFateStore<>(zk, null, null); - ReadOnlyFateStore readOnlyUFS = - new UserFateStore<>(context, SystemTables.FATE.tableName(), null, null); - return Map.of(FateInstanceType.META, readOnlyMFS, FateInstanceType.USER, readOnlyUFS); - } - - private ServiceLock createAdminLock(ServerContext context) throws InterruptedException { - var zk = context.getZooSession(); - UUID uuid = UUID.randomUUID(); - ServiceLockPath slp = context.getServerPaths().createAdminLockPath(); - ServiceLock adminLock = new ServiceLock(zk, slp, uuid); - AdminLockWatcher lw = new AdminLockWatcher(); - ServiceLockData.ServiceDescriptors descriptors = new ServiceLockData.ServiceDescriptors(); - descriptors.addService(new ServiceLockData.ServiceDescriptor(uuid, - ServiceLockData.ThriftService.NONE, "fake_admin_util_host", ResourceGroupId.DEFAULT)); - ServiceLockData sld = new ServiceLockData(descriptors); - String lockPath = slp.toString(); - String parentLockPath = lockPath.substring(0, lockPath.lastIndexOf("/")); - - try { - var zrw = zk.asReaderWriter(); - zrw.putPersistentData(parentLockPath, new byte[0], ZooUtil.NodeExistsPolicy.SKIP); - zrw.putPersistentData(lockPath, new byte[0], ZooUtil.NodeExistsPolicy.SKIP); - } catch (KeeperException | InterruptedException e) { - throw new IllegalStateException("Error creating path in ZooKeeper", e); - } - - adminLock.lock(lw, sld); - lockAcquiredLatch.await(); - - return adminLock; - } - - private void validateFateUserInput(FateOpsCommand cmd) { - if (cmd.cancel && cmd.fail || cmd.cancel && cmd.delete || cmd.fail && cmd.delete) { - throw new IllegalArgumentException( - "Can only perform one of the following at a time: cancel, fail or delete."); - } - if ((cmd.cancel || cmd.fail || cmd.delete) && cmd.fateIdList.isEmpty()) { - throw new IllegalArgumentException( - "At least one txId required when using cancel, fail or delete"); - } - } - - private void cancelSubmittedFateTxs(ServerContext context, List fateIdList) - throws AccumuloException { - for (String fateIdStr : fateIdList) { - FateId fateId = FateId.from(fateIdStr); - TFateId thriftFateId = fateId.toThrift(); - boolean cancelled = cancelFateOperation(context, thriftFateId); - if (cancelled) { - System.out.println("FaTE transaction " + fateId + " was cancelled or already completed."); - } else { - System.out - .println("FaTE transaction " + fateId + " was not cancelled, status may have changed."); - } - } - } - - private boolean cancelFateOperation(ClientContext context, TFateId thriftFateId) - throws AccumuloException { - FateService.Client client = null; - try { - client = ThriftClientTypes.FATE.getConnectionWithRetry(context); - return client.cancelFateOperation(TraceUtil.traceInfo(), context.rpcCreds(), thriftFateId); - } catch (Exception e) { - throw new AccumuloException(e); - } finally { - if (client != null) { - ThriftUtil.close(client, context); - } - } - } - - private void summarizeFateTx(ServerContext context, FateOpsCommand cmd, AdminUtil admin, - Map> fateStores, ServiceLockPath tableLocksPath) - throws InterruptedException, AccumuloException, AccumuloSecurityException, KeeperException, - NamespaceNotFoundException { - - var zk = context.getZooSession(); - var transactions = admin.getStatus(fateStores, zk, tableLocksPath, null, null, null); - - // build id map - relies on unique ids for tables and namespaces - // used to look up the names of either table or namespace by id. - Map tidToNameMap = context.createTableIdToQualifiedNameMap(); - Map idsToNameMap = new HashMap<>(tidToNameMap.size() * 2); - tidToNameMap.forEach((tid, name) -> idsToNameMap.put(tid.canonical(), "t:" + name)); - context.namespaceOperations().namespaceIdMap().forEach((name, nsid) -> { - String prev = idsToNameMap.put(nsid, "ns:" + name); - if (prev != null) { - log.warn("duplicate id found for table / namespace id. table name: {}, namespace name: {}", - prev, name); - } - }); - - Set fateIdFilter = - cmd.fateIdList.stream().map(FateId::from).collect(Collectors.toSet()); - EnumSet statusFilter = getCmdLineStatusFilters(cmd.states); - EnumSet typesFilter = getCmdLineInstanceTypeFilters(cmd.instanceTypes); - - FateSummaryReport report = - new FateSummaryReport(idsToNameMap, fateIdFilter, statusFilter, typesFilter); - - // gather statistics - transactions.getTransactions().forEach(report::gatherTxnStatus); - if (cmd.printJson) { - printLines(Collections.singletonList(report.toJson())); - } else { - printLines(report.formatLines()); - } - } - - private void printLines(List lines) { - for (String nextLine : lines) { - if (nextLine == null) { - continue; - } - System.out.println(nextLine); - } - } - - /** - * If provided on the command line, get the TStatus values provided. - * - * @return a set of status filters, or null if none provided - */ - private EnumSet getCmdLineStatusFilters(List states) { - EnumSet statusFilter = null; - if (!states.isEmpty()) { - statusFilter = EnumSet.noneOf(ReadOnlyFateStore.TStatus.class); - for (String element : states) { - statusFilter.add(ReadOnlyFateStore.TStatus.valueOf(element)); - } - } - return statusFilter; - } - - /** - * If provided on the command line, get the FateInstanceType values provided. - * - * @return a set of fate instance types filters, or null if none provided - */ - private EnumSet getCmdLineInstanceTypeFilters(List instanceTypes) { - EnumSet typesFilter = null; - if (!instanceTypes.isEmpty()) { - typesFilter = EnumSet.noneOf(FateInstanceType.class); - for (String instanceType : instanceTypes) { - typesFilter.add(FateInstanceType.valueOf(instanceType)); - } - } - return typesFilter; - } - - /** - * Finds tablets that point to fate operations that do not exists or are complete. - * - * @param tablets the tablets to inspect - * @param tabletLookup a function that can lookup a tablets latest metadata - * @param activePredicate a predicate that can determine if a fate id is currently active - * @param danglingConsumer a consumer that tablets with inactive fate ids will be sent to - */ - static void findDanglingFateOperations(Iterable tablets, - Function,Map> tabletLookup, - Predicate activePredicate, BiConsumer> danglingConsumer, - int bufferSize) { - - ArrayList fateIds = new ArrayList<>(); - Map> candidates = new HashMap<>(); - for (TabletMetadata tablet : tablets) { - fateIds.clear(); - getAllFateIds(tablet, fateIds::add); - fateIds.removeIf(activePredicate); - if (!fateIds.isEmpty()) { - candidates.put(tablet.getExtent(), new HashSet<>(fateIds)); - if (candidates.size() > bufferSize) { - processCandidates(candidates, tabletLookup, danglingConsumer); - candidates.clear(); - } - } - } - - processCandidates(candidates, tabletLookup, danglingConsumer); - } - - private static void processCandidates(Map> candidates, - Function,Map> tabletLookup, - BiConsumer> danglingConsumer) { - // Perform a 2nd check of the tablet to avoid race conditions like the following. - // 1. THREAD 1 : TabletMetadata is read and points to active fate operation - // 2. THREAD 2 : The fate operation is deleted from the tablet - // 3. THREAD 2 : The fate operation completes - // 4. THREAD 1 : Checks if the fate operation read in step 1 is active and finds it is not - - Map currentTablets = tabletLookup.apply(candidates.keySet()); - HashSet currentFateIds = new HashSet<>(); - candidates.forEach((extent, fateIds) -> { - var currentTablet = currentTablets.get(extent); - if (currentTablet != null) { - currentFateIds.clear(); - getAllFateIds(currentTablet, currentFateIds::add); - // Only keep fate ids that are still present in the tablet. Any new fate ids in - // currentFateIds that were not seen on the first pass are not considered here. To check - // those new ones, the entire two-step process would need to be rerun. - fateIds.retainAll(currentFateIds); - - if (!fateIds.isEmpty()) { - // the fateIds in this set were found to be inactive and still exist in the tablet - // metadata after being found inactive - danglingConsumer.accept(extent, fateIds); - } - } // else the tablet no longer exist so nothing to report - }); - } - - /** - * Extracts all fate ids that a tablet points to from any field. - */ - private static void getAllFateIds(TabletMetadata tabletMetadata, - Consumer fateIdConsumer) { - tabletMetadata.getLoaded().values().forEach(fateIdConsumer); - if (tabletMetadata.getSelectedFiles() != null) { - fateIdConsumer.accept(tabletMetadata.getSelectedFiles().getFateId()); - } - if (tabletMetadata.getOperationId() != null) { - fateIdConsumer.accept(tabletMetadata.getOperationId().getFateId()); - } - } - - @VisibleForTesting - public static int executeCheckCommand(ServerContext context, CheckCommand cmd, - ServerUtilOpts opts) throws Exception { - validateAndTransformCheckCommand(cmd); - - if (cmd.list) { - listChecks(); - } else if (cmd.run) { - var givenChecks = cmd.checks.stream() - .map(name -> CheckCommand.Check.valueOf(name.toUpperCase())).collect(Collectors.toList()); - return executeRunCheckCommand(cmd, givenChecks, context, opts); - } - - return 0; - } - - private static void validateAndTransformCheckCommand(CheckCommand cmd) { - Preconditions.checkArgument(cmd.list != cmd.run, "Must use either 'list' or 'run'"); - if (cmd.list) { - Preconditions.checkArgument(cmd.checks == null && cmd.pattern == null, - "'list' does not expect any further arguments"); - } else if (cmd.pattern != null) { - Preconditions.checkArgument(cmd.checks == null, "Expected one argument (the regex pattern)"); - List matchingChecks = new ArrayList<>(); - var pattern = Pattern.compile(cmd.pattern.toUpperCase()); - for (CheckCommand.Check check : CheckCommand.Check.values()) { - if (pattern.matcher(check.name()).matches()) { - matchingChecks.add(check.name()); - } - } - Preconditions.checkArgument(!matchingChecks.isEmpty(), - "No checks matched the given pattern: " + pattern.pattern()); - cmd.checks = matchingChecks; - } else { - if (cmd.checks == null) { - cmd.checks = EnumSet.allOf(CheckCommand.Check.class).stream().map(Enum::name) - .collect(Collectors.toList()); - } - } - } - - private static void listChecks() { - System.out.println(); - System.out.printf("%-20s | %-90s | %-20s%n", "Check Name", "Description", "Depends on"); - System.out.println("-".repeat(130)); - for (CheckCommand.Check check : CheckCommand.Check.values()) { - System.out.printf("%-20s | %-90s | %-20s%n", check.name(), check.getDescription(), - check.getDependencies().stream().map(CheckCommand.Check::name) - .collect(Collectors.joining(", "))); - } - System.out.println("-".repeat(130)); - System.out.println(); - } - - private static int executeRunCheckCommand(CheckCommand cmd, List givenChecks, - ServerContext context, ServerUtilOpts opts) throws Exception { - // Get all the checks in the order they are declared in the enum - final var allChecks = CheckCommand.Check.values(); - final TreeMap checkStatus = new TreeMap<>(); - - for (CheckCommand.Check check : allChecks) { - if (depsFailed(check, checkStatus)) { - checkStatus.put(check, CheckCommand.CheckStatus.SKIPPED_DEPENDENCY_FAILED); - } else { - if (givenChecks.contains(check)) { - checkStatus.put(check, cmd.getCheckRunner(check).runCheck(context, opts, cmd.fixFiles)); - } else { - checkStatus.put(check, CheckCommand.CheckStatus.FILTERED_OUT); - } - } - } - - printChecksResults(checkStatus); - - if (checkStatus.values().stream() - .anyMatch(status -> status == CheckCommand.CheckStatus.FAILED)) { - return 5; - } else { - return 0; - } - } - - private static boolean depsFailed(CheckCommand.Check check, - TreeMap checkStatus) { - return check.getDependencies().stream() - .anyMatch(dep -> checkStatus.get(dep) == CheckCommand.CheckStatus.FAILED - || checkStatus.get(dep) == CheckCommand.CheckStatus.SKIPPED_DEPENDENCY_FAILED); - } - - private static void - printChecksResults(TreeMap checkStatus) { - System.out.println(); - System.out.printf("%-20s | %-20s%n", "Check Name", "Status"); - System.out.println("-".repeat(50)); - for (Map.Entry entry : checkStatus.entrySet()) { - System.out.printf("%-20s | %-20s%n", entry.getKey().name(), entry.getValue().name()); - } - System.out.println("-".repeat(50)); - System.out.println(); - } - - /** - * Wrapper around the fate stores - */ - private static class FateStores implements AutoCloseable { - private final Map> storesMap; - - private FateStores(FateInstanceType type1, FateStore store1, FateInstanceType type2, - FateStore store2) { - storesMap = Map.of(type1, store1, type2, store2); - } - - private Map> getStoresMap() { - return storesMap; - } - - @Override - public void close() { - for (var fs : storesMap.values()) { - fs.close(); - } - } - } -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ServerKeywordExecutable.java b/server/base/src/main/java/org/apache/accumulo/server/util/ServerKeywordExecutable.java new file mode 100644 index 00000000000..406afd2db2f --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ServerKeywordExecutable.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.security.SecurityUtil; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; + +public abstract class ServerKeywordExecutable + implements KeywordExecutable { + + private final O options; + + public ServerKeywordExecutable(O options) { + this.options = options; + } + + @Override + public final void execute(String[] args) throws Exception { + JCommander cl = new JCommander(this.options); + cl.setProgramName("accumulo " + usageGroup().name().toLowerCase() + " " + keyword()); + cl.parse(args); + + if (this.options.help) { + cl.usage(); + return; + } + // Login as the server on secure HDFS + try (ServerContext context = options.getServerContext()) { + AccumuloConfiguration conf = options.getServerContext().getConfiguration(); + if (conf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) { + SecurityUtil.serverLogin(conf); + } + execute(cl, options); + } + } + + public abstract void execute(JCommander cl, O options) throws Exception; +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java index 8b84b2099ed..d20d517599f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java @@ -237,8 +237,8 @@ private static void removeSingletonLock(ZooReaderWriter zoo, ServiceLockPath pat } } - static Optional filterSingleton(ServerContext context, ServiceLockPath path, - AddressSelector addressSelector) { + public static Optional filterSingleton(ServerContext context, + ServiceLockPath path, AddressSelector addressSelector) { Optional sld = context.getZooCache().getLockData(path); return sld.filter(lockData -> { for (var service : ServiceLockData.ThriftService.values()) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ChangeSecret.java similarity index 82% rename from server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ChangeSecret.java index 5538e875f7c..c8918c4a703 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ChangeSecret.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; @@ -27,7 +27,6 @@ import java.util.UUID; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; @@ -38,7 +37,10 @@ import org.apache.accumulo.core.zookeeper.ZooSession; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServerDirs; +import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -50,16 +52,40 @@ import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; +import com.beust.jcommander.JCommander; +import com.google.auto.service.AutoService; + import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; -public class ChangeSecret { +@AutoService(KeywordExecutable.class) +public class ChangeSecret extends ServerKeywordExecutable { + + public ChangeSecret() { + super(new ServerUtilOpts()); + } + + @Override + public String keyword() { + return "change-secret"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Changes the unique secret given to the instance that all servers must know."; + } - public static void execute(final ServerContext context, final AccumuloConfiguration conf) - throws Exception { + @Override + public void execute(JCommander cl, ServerUtilOpts options) throws Exception { + ServerContext context = options.getServerContext(); try (var fs = context.getVolumeManager()) { - ServerDirs serverDirs = new ServerDirs(conf, new Configuration()); + ServerDirs serverDirs = new ServerDirs(context.getConfiguration(), new Configuration()); verifyHdfsWritePermission(serverDirs, fs); String oldPass = String.valueOf(System.console().readPassword("Old secret: ")); @@ -84,11 +110,11 @@ public static void execute(final ServerContext context, final AccumuloConfigurat } } - interface Visitor { + private interface Visitor { void visit(ZooReader zoo, String path) throws Exception; } - private static void recurse(ZooReader zoo, String root, Visitor v) { + private void recurse(ZooReader zoo, String root, Visitor v) { try { v.visit(zoo, root); for (String child : zoo.getChildren(root)) { @@ -99,8 +125,7 @@ private static void recurse(ZooReader zoo, String root, Visitor v) { } } - private static void verifyAccumuloIsDown(ServerContext context, String oldPassword) - throws Exception { + private void verifyAccumuloIsDown(ServerContext context, String oldPassword) throws Exception { var conf = context.getSiteConfiguration(); try (var oldZk = new ZooSession(ChangeSecret.class.getSimpleName() + ".verifyAccumuloIsDown(oldPassword)", @@ -123,8 +148,8 @@ private static void verifyAccumuloIsDown(ServerContext context, String oldPasswo } } - private static void rewriteZooKeeperInstance(final ServerContext context, - final InstanceId newInstanceId, String oldPass, String newPass) throws Exception { + private void rewriteZooKeeperInstance(final ServerContext context, final InstanceId newInstanceId, + String oldPass, String newPass) throws Exception { var conf = context.getSiteConfiguration(); try ( var oldZk = new ZooSession( @@ -169,7 +194,7 @@ private static void rewriteZooKeeperInstance(final ServerContext context, } } - private static void updateHdfs(ServerDirs serverDirs, VolumeManager fs, InstanceId newInstanceId) + private void updateHdfs(ServerDirs serverDirs, VolumeManager fs, InstanceId newInstanceId) throws IOException { // Need to recreate the instanceId on all of them to keep consistency for (Volume v : fs.getVolumes()) { @@ -186,8 +211,7 @@ private static void updateHdfs(ServerDirs serverDirs, VolumeManager fs, Instance } } - private static void verifyHdfsWritePermission(ServerDirs serverDirs, VolumeManager fs) - throws Exception { + private void verifyHdfsWritePermission(ServerDirs serverDirs, VolumeManager fs) throws Exception { for (Volume v : fs.getVolumes()) { final Path instanceId = serverDirs.getInstanceIdLocation(v); FileStatus fileStatus = v.getFileSystem().getFileStatus(instanceId); @@ -195,7 +219,7 @@ private static void verifyHdfsWritePermission(ServerDirs serverDirs, VolumeManag } } - private static void checkHdfsAccessPermissions(FileStatus stat, FsAction mode) throws Exception { + private void checkHdfsAccessPermissions(FileStatus stat, FsAction mode) throws Exception { FsPermission perm = stat.getPermission(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user = ugi.getShortUserName(); @@ -217,7 +241,7 @@ private static void checkHdfsAccessPermissions(FileStatus stat, FsAction mode) t stat.getPath(), stat.getOwner(), stat.getGroup(), stat.isDirectory() ? "d" : "-", perm)); } - private static void deleteInstance(ServerContext context, String oldPass) throws Exception { + private void deleteInstance(ServerContext context, String oldPass) throws Exception { var conf = context.getSiteConfiguration(); try (var oldZk = new ZooSession(ChangeSecret.class.getSimpleName() + ".deleteInstance()", conf.get(Property.INSTANCE_ZK_HOST), diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DeleteZooInstance.java similarity index 66% rename from server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DeleteZooInstance.java index 2b94c45496a..ed87f3d76b2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/DeleteZooInstance.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DeleteZooInstance.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; @@ -29,36 +29,80 @@ import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.DeleteZooInstance.DeleteZooInstanceOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DeleteZooInstance { +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class DeleteZooInstance extends ServerKeywordExecutable { private static final Logger log = LoggerFactory.getLogger(DeleteZooInstance.class); - public static void execute(final ServerContext context, final boolean clean, - final String instance, final String auth) throws InterruptedException, KeeperException { + static class DeleteZooInstanceOpts extends ServerUtilOpts { + @Parameter(names = {"-i", "--instance"}, description = "the instance name or id to delete") + String instance; + + @Parameter(names = {"-c", "--clean"}, + description = "Cleans Zookeeper by deleting all old instances. This will not delete the instance pointed to by the local accumulo.properties file") + boolean clean = false; + + @Parameter(names = {"--password"}, + description = "The system secret, if different than instance.secret in accumulo.properties", + password = true) + String auth; + } + + public DeleteZooInstance() { + super(new DeleteZooInstanceOpts()); + } + + @Override + public String keyword() { + return "delete-instance"; + } - if (auth != null) { - context.getZooSession().addAccumuloDigestAuth(auth); + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Deletes specific instance name or id from zookeeper or cleans up all old instances."; + } + + @Override + public void execute(JCommander cl, DeleteZooInstanceOpts options) throws Exception { + + ServerContext context = options.getServerContext(); + + if (options.auth != null) { + context.getZooSession().addAccumuloDigestAuth(options.auth); } - if (clean) { + if (options.clean) { // If clean is set to true then a specific instance should not be set - if (instance != null) { + if (options.instance != null) { throw new IllegalArgumentException( "Cannot set clean flag to true and also an instance name"); } cleanAllOld(context); } else { // If all old is false then we require a specific instance - Objects.requireNonNull(instance, "Instance name must not be null"); - removeInstance(context, instance); + Objects.requireNonNull(options.instance, "Instance name must not be null"); + removeInstance(context, options.instance); } } - private static void removeInstance(ServerContext context, final String instance) + private void removeInstance(ServerContext context, final String instance) throws InterruptedException, KeeperException { var zrw = context.getZooSession().asReaderWriter(); // try instance name: @@ -93,8 +137,7 @@ private static void removeInstance(ServerContext context, final String instance) } } - private static void cleanAllOld(ServerContext context) - throws InterruptedException, KeeperException { + private void cleanAllOld(ServerContext context) throws InterruptedException, KeeperException { var zrw = context.getZooSession().asReaderWriter(); for (String child : zrw.getChildren(Constants.ZROOT)) { if (Constants.ZINSTANCES.equals("/" + child)) { @@ -112,7 +155,7 @@ private static void cleanAllOld(ServerContext context) } } - private static boolean checkCurrentInstance(ServerContext context, String instanceName, + private boolean checkCurrentInstance(ServerContext context, String instanceName, String instanceId) { boolean operate = true; // If the instance given is the current instance we should verify the user actually wants to @@ -128,20 +171,20 @@ private static boolean checkCurrentInstance(ServerContext context, String instan return operate; } - private static String getRootChildPath(String child) { + private String getRootChildPath(String child) { return Constants.ZROOT + "/" + child; } - private static String getInstancePath(final String instanceName) { + private String getInstancePath(final String instanceName) { return Constants.ZROOT + Constants.ZINSTANCES + "/" + instanceName; } - private static List getInstances(final ZooReaderWriter zk) + private List getInstances(final ZooReaderWriter zk) throws InterruptedException, KeeperException { return zk.getChildren(Constants.ZROOT + Constants.ZINSTANCES); } - private static void deleteRetry(ZooReaderWriter zk, String path) + private void deleteRetry(ZooReaderWriter zk, String path) throws InterruptedException, KeeperException { for (int i = 0; i < 10; i++) { try { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DumpConfig.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DumpConfig.java new file mode 100644 index 00000000000..45d2f68db53 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/DumpConfig.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedSet; +import java.util.TreeMap; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.NamespaceNotFoundException; +import org.apache.accumulo.core.client.ResourceGroupNotFoundException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.NamespacePermission; +import org.apache.accumulo.core.security.SystemPermission; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.DumpConfig.DumpConfigOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Lists; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +@AutoService(KeywordExecutable.class) +public class DumpConfig extends ServerKeywordExecutable { + + // This only exists because it is called from DumpConfigIT + public static void main(String[] args) throws Exception { + new DumpConfig().execute(args); + } + + static class DumpConfigOpts extends ServerUtilOpts { + @Parameter(names = {"-a", "--all"}, + description = "print the system and all table configurations") + boolean allConfiguration = false; + + @Parameter(names = {"-d", "--directory"}, description = "directory to place config files") + String directory = null; + + @Parameter(names = {"-s", "--system"}, description = "print the system configuration") + boolean systemConfiguration = false; + + @Parameter(names = {"-rg", "--resourceGroups"}, + description = "print the resource group configuration") + boolean resourceGroupConfiguration = false; + + @Parameter(names = {"-n", "--namespaces"}, description = "print the namespace configuration") + boolean namespaceConfiguration = false; + + @Parameter(names = {"-t", "--tables"}, description = "print per-table configuration") + List tables = new ArrayList<>(); + + @Parameter(names = {"-u", "--users"}, + description = "print users and their authorizations and permissions") + boolean users = false; + } + + private static final String ACCUMULO_SITE_BACKUP_FILE = "accumulo.properties.bak"; + private static final String NS_FILE_SUFFIX = "_ns.cfg"; + private static final String RG_FILE_SUFFIX = "_rg.cfg"; + private static final String USER_FILE_SUFFIX = "_user.cfg"; + private static final MessageFormat configFormat = new MessageFormat("config -t {0} -s {1}\n"); + private static final MessageFormat createNsFormat = new MessageFormat("createnamespace {0}\n"); + private static final MessageFormat createTableFormat = new MessageFormat("createtable {0}\n"); + private static final MessageFormat createUserFormat = new MessageFormat("createuser {0}\n"); + private static final MessageFormat createRGFormat = + new MessageFormat("createresourcegroup {0}\n"); + private static final MessageFormat nsConfigFormat = new MessageFormat("config -ns {0} -s {1}\n"); + private static final MessageFormat rgConfigFormat = new MessageFormat("config -rg {0} -s {1}\n"); + private static final MessageFormat sysPermFormat = + new MessageFormat("grant System.{0} -s -u {1}\n"); + private static final MessageFormat nsPermFormat = + new MessageFormat("grant Namespace.{0} -ns {1} -u {2}\n"); + private static final MessageFormat tablePermFormat = + new MessageFormat("grant Table.{0} -t {1} -u {2}\n"); + private static final MessageFormat userAuthsFormat = + new MessageFormat("setauths -u {0} -s {1}\n"); + + public DumpConfig() { + super(new DumpConfigOpts()); + } + + @Override + public String keyword() { + return "dump-config"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Prints out non-default configuration settings."; + } + + @Override + public void execute(JCommander cl, DumpConfigOpts opts) throws Exception { + + ServerContext context = opts.getServerContext(); + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "app is run in same security context as user providing the filename") + File outputDirectory = getOutputDirectory(opts.directory); + DefaultConfiguration defaultConfig = DefaultConfiguration.getInstance(); + Map siteConfig = context.instanceOperations().getSiteConfiguration(); + Map systemConfig = context.instanceOperations().getSystemConfiguration(); + List localUsers = null; + if (opts.allConfiguration || opts.users) { + localUsers = Lists.newArrayList(context.securityOperations().listLocalUsers()); + Collections.sort(localUsers); + } + + if (opts.allConfiguration) { + // print accumulo site + printSystemConfiguration(outputDirectory, systemConfig, siteConfig, defaultConfig); + // print resource groups + for (ResourceGroupId group : context.resourceGroupOperations().list()) { + printResourceGroupConfiguration(context, group, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + // print namespaces + for (String namespace : context.namespaceOperations().list()) { + printNameSpaceConfiguration(context, namespace, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + // print tables + SortedSet tableNames = context.tableOperations().list(); + for (String tableName : tableNames) { + printTableConfiguration(context, tableName, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + // print users + if (localUsers != null) { + for (String user : localUsers) { + printUserConfiguration(context, user, outputDirectory); + } + } + } else { + if (opts.systemConfiguration) { + printSystemConfiguration(outputDirectory, systemConfig, siteConfig, defaultConfig); + } + if (opts.resourceGroupConfiguration) { + for (ResourceGroupId group : context.resourceGroupOperations().list()) { + printResourceGroupConfiguration(context, group, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + } + if (opts.namespaceConfiguration) { + for (String namespace : context.namespaceOperations().list()) { + printNameSpaceConfiguration(context, namespace, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + } + if (!opts.tables.isEmpty()) { + for (String tableName : opts.tables) { + printTableConfiguration(context, tableName, outputDirectory, systemConfig, siteConfig, + defaultConfig); + } + } + if (opts.users && localUsers != null) { + for (String user : localUsers) { + printUserConfiguration(context, user, outputDirectory); + } + } + } + } + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "app is run in same security context as user providing the filename") + private File getOutputDirectory(final String directory) { + if (directory == null) { + return null; + } + Path outputDirectory = Path.of(directory); + if (!Files.isDirectory(outputDirectory)) { + throw new IllegalArgumentException(directory + " does not exist on the local filesystem."); + } + if (!Files.isWritable(outputDirectory)) { + throw new IllegalArgumentException(directory + " is not writable"); + } + return outputDirectory.toFile(); + } + + private String getDefaultConfigValue(DefaultConfiguration defaultConfig, String key) { + if (key == null) { + return null; + } + + String defaultValue = null; + try { + Property p = Property.getPropertyByKey(key); + if (p == null) { + return defaultValue; + } + defaultValue = defaultConfig.get(p); + } catch (IllegalArgumentException e) { + // ignore + } + return defaultValue; + } + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "code runs in same security context as user who provided input") + private void printResourceGroupConfiguration(AccumuloClient accumuloClient, ResourceGroupId group, + File outputDirectory, Map systemConfig, Map siteConfig, + DefaultConfiguration defaultConfig) throws IOException, AccumuloException, + AccumuloSecurityException, ResourceGroupNotFoundException { + Path rgScript = outputDirectory.toPath().resolve(group + RG_FILE_SUFFIX); + try (BufferedWriter nsWriter = Files.newBufferedWriter(rgScript)) { + nsWriter.write(createRGFormat.format(new String[] {group.canonical()})); + Map props = + ImmutableSortedMap.copyOf(accumuloClient.resourceGroupOperations().getProperties(group)); + for (Entry entry : props.entrySet()) { + String defaultValue = getDefaultConfigValue(defaultConfig, entry.getKey()); + if (defaultValue == null || !defaultValue.equals(entry.getValue())) { + if (!entry.getValue().equals(siteConfig.get(entry.getKey())) + && !entry.getValue().equals(systemConfig.get(entry.getKey()))) { + nsWriter.write(rgConfigFormat + .format(new String[] {group.canonical(), entry.getKey() + "=" + entry.getValue()})); + } + } + } + } + } + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "code runs in same security context as user who provided input") + private void printNameSpaceConfiguration(AccumuloClient accumuloClient, String namespace, + File outputDirectory, Map systemConfig, Map siteConfig, + DefaultConfiguration defaultConfig) + throws IOException, AccumuloException, AccumuloSecurityException, NamespaceNotFoundException { + Path namespaceScript = outputDirectory.toPath().resolve(namespace + NS_FILE_SUFFIX); + try (BufferedWriter nsWriter = Files.newBufferedWriter(namespaceScript)) { + nsWriter.write(createNsFormat.format(new String[] {namespace})); + Map props = ImmutableSortedMap + .copyOf(accumuloClient.namespaceOperations().getConfiguration(namespace)); + for (Entry entry : props.entrySet()) { + String defaultValue = getDefaultConfigValue(defaultConfig, entry.getKey()); + if (defaultValue == null || !defaultValue.equals(entry.getValue())) { + if (!entry.getValue().equals(siteConfig.get(entry.getKey())) + && !entry.getValue().equals(systemConfig.get(entry.getKey()))) { + nsWriter.write(nsConfigFormat + .format(new String[] {namespace, entry.getKey() + "=" + entry.getValue()})); + } + } + } + } + } + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "code runs in same security context as user who provided input") + private void printUserConfiguration(AccumuloClient accumuloClient, String user, + File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException { + Path userScript = outputDirectory.toPath().resolve(user + USER_FILE_SUFFIX); + try (BufferedWriter userWriter = Files.newBufferedWriter(userScript)) { + userWriter.write(createUserFormat.format(new String[] {user})); + Authorizations auths = accumuloClient.securityOperations().getUserAuthorizations(user); + userWriter.write(userAuthsFormat.format(new String[] {user, auths.toString()})); + for (SystemPermission sp : SystemPermission.values()) { + if (accumuloClient.securityOperations().hasSystemPermission(user, sp)) { + userWriter.write(sysPermFormat.format(new String[] {sp.name(), user})); + } + } + for (String namespace : accumuloClient.namespaceOperations().list()) { + for (NamespacePermission np : NamespacePermission.values()) { + if (accumuloClient.securityOperations().hasNamespacePermission(user, namespace, np)) { + userWriter.write(nsPermFormat.format(new String[] {np.name(), namespace, user})); + } + } + } + for (String tableName : accumuloClient.tableOperations().list()) { + for (TablePermission perm : TablePermission.values()) { + if (accumuloClient.securityOperations().hasTablePermission(user, tableName, perm)) { + userWriter.write(tablePermFormat.format(new String[] {perm.name(), tableName, user})); + } + } + } + } + } + + private void printSystemConfiguration(File outputDirectory, Map systemConfig, + Map siteConfig, DefaultConfiguration defaultConfig) throws IOException { + TreeMap conf = new TreeMap<>(); + TreeMap site = new TreeMap<>(siteConfig); + for (Entry prop : site.entrySet()) { + String defaultValue = getDefaultConfigValue(defaultConfig, prop.getKey()); + if (!prop.getValue().equals(defaultValue) && !systemConfig.containsKey(prop.getKey())) { + conf.put(prop.getKey(), prop.getValue()); + } + } + TreeMap system = new TreeMap<>(systemConfig); + for (Entry prop : system.entrySet()) { + String defaultValue = getDefaultConfigValue(defaultConfig, prop.getKey()); + if (!prop.getValue().equals(defaultValue)) { + conf.put(prop.getKey(), prop.getValue()); + } + } + Path siteBackup = outputDirectory.toPath().resolve(ACCUMULO_SITE_BACKUP_FILE); + try (BufferedWriter fw = Files.newBufferedWriter(siteBackup)) { + for (Entry prop : conf.entrySet()) { + fw.write(prop.getKey() + "=" + prop.getValue() + "\n"); + } + } + } + + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", + justification = "code runs in same security context as user who provided input") + private void printTableConfiguration(AccumuloClient accumuloClient, String tableName, + File outputDirectory, Map systemConfig, Map siteConfig, + DefaultConfiguration defaultConfig) + throws AccumuloException, TableNotFoundException, IOException { + Path tableBackup = outputDirectory.toPath().resolve(tableName + ".cfg"); + try (BufferedWriter writer = Files.newBufferedWriter(tableBackup)) { + writer.write(createTableFormat.format(new String[] {tableName})); + Map props = + ImmutableSortedMap.copyOf(accumuloClient.tableOperations().getConfiguration(tableName)); + for (Entry prop : props.entrySet()) { + if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) { + String defaultValue = getDefaultConfigValue(defaultConfig, prop.getKey()); + if (defaultValue == null || !defaultValue.equals(prop.getValue())) { + if (!prop.getValue().equals(siteConfig.get(prop.getKey())) + && !prop.getValue().equals(systemConfig.get(prop.getKey()))) { + writer.write(configFormat + .format(new String[] {tableName, prop.getKey() + "=" + prop.getValue()})); + } + } + } + } + } + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Fate.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Fate.java new file mode 100644 index 00000000000..b8f0c7bbd4a --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Fate.java @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Formatter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.NamespaceNotFoundException; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.fate.AdminUtil; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.FateStore; +import org.apache.accumulo.core.fate.ReadOnlyFateStore; +import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; +import org.apache.accumulo.core.manager.thrift.FateService; +import org.apache.accumulo.core.manager.thrift.TFateId; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.Halt; +import org.apache.accumulo.core.zookeeper.ZooSession; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.Fate.FateOpts; +import org.apache.accumulo.server.util.fateCommand.FateSummaryReport; +import org.apache.accumulo.start.spi.KeywordExecutable; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class Fate extends ServerKeywordExecutable { + + private static final Logger LOG = LoggerFactory.getLogger(Fate.class); + + // This only exists because it is called from ITs + public static void main(String[] args) throws Exception { + new Fate().execute(args); + } + + /** + * Wrapper around the fate stores + */ + private static class FateStores implements AutoCloseable { + private final Map> storesMap; + + private FateStores(FateInstanceType type1, FateStore store1, FateInstanceType type2, + FateStore store2) { + storesMap = Map.of(type1, store1, type2, store2); + } + + private Map> getStoresMap() { + return storesMap; + } + + @Override + public void close() { + for (var fs : storesMap.values()) { + fs.close(); + } + } + } + + static class FateOpts extends ServerUtilOpts { + @Parameter(description = "[...]") + List fateIdList = new ArrayList<>(); + + @Parameter(names = {"-c", "--cancel"}, + description = "... Cancel new or submitted FaTE transactions") + boolean cancel; + + @Parameter(names = {"-f", "--fail"}, + description = "... Transition FaTE transaction status to FAILED_IN_PROGRESS") + boolean fail; + + @Parameter(names = {"-d", "--delete"}, + description = "... Delete FaTE transaction and its associated table locks") + boolean delete; + + @Parameter(names = {"-p", "--print", "-print", "-l", "--list", "-list"}, + description = "[...] Print information about FaTE transactions. Print only the FateId's specified or print all transactions if empty. Use -s to only print those with certain states. Use -t to only print those with certain FateInstanceTypes.") + boolean print; + + @Parameter(names = "--summary", + description = "[...] Print a summary of FaTE transactions. Print only the FateId's specified or print all transactions if empty. Use -s to only print those with certain states. Use -t to only print those with certain FateInstanceTypes. Use -j to print the transactions in json.") + boolean summarize; + + @Parameter(names = {"-j", "--json"}, + description = "Print transactions in json. Only useful for --summary command.") + boolean printJson; + + @Parameter(names = {"-s", "--state"}, + description = "... Print transactions in the state(s) {NEW, IN_PROGRESS, FAILED_IN_PROGRESS, FAILED, SUCCESSFUL}") + List states = new ArrayList<>(); + + @Parameter(names = {"-t", "--type"}, + description = "... Print transactions of fate instance type(s) {USER, META}") + List instanceTypes = new ArrayList<>(); + } + + private final CountDownLatch lockAcquiredLatch = new CountDownLatch(1); + + private class AdminLockWatcher implements ServiceLock.AccumuloLockWatcher { + @Override + public void lostLock(ServiceLock.LockLossReason reason) { + String msg = "Admin lost lock: " + reason.toString(); + if (reason == ServiceLock.LockLossReason.LOCK_DELETED) { + Halt.halt(0, msg); + } else { + Halt.halt(1, msg); + } + } + + @Override + public void unableToMonitorLockNode(Exception e) { + String msg = "Admin unable to monitor lock: " + e.getMessage(); + LOG.warn(msg); + Halt.halt(1, msg); + } + + @Override + public void acquiredLock() { + lockAcquiredLatch.countDown(); + LOG.debug("Acquired ZooKeeper lock for Admin"); + } + + @Override + public void failedToAcquireLock(Exception e) { + LOG.warn("Failed to acquire ZooKeeper lock for Admin, msg: " + e.getMessage()); + } + } + + public Fate() { + super(new FateOpts()); + } + + @Override + public String keyword() { + return "fate"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Operations performed on the Manager FaTE system."; + } + + @Override + public void execute(JCommander cl, FateOpts options) throws Exception { + + ServerContext context = options.getServerContext(); + validateFateUserInput(options); + + AdminUtil admin = new AdminUtil<>(); + var zTableLocksPath = context.getServerPaths().createTableLocksPath(); + var zk = context.getZooSession(); + ServiceLock adminLock = null; + Map> readOnlyFateStores = null; + + try { + if (options.cancel) { + cancelSubmittedFateTxs(context, options.fateIdList); + } else if (options.fail) { + adminLock = createAdminLock(context); + try (var fateStores = createFateStores(context, zk, adminLock)) { + for (String fateIdStr : options.fateIdList) { + if (!admin.prepFail(fateStores.getStoresMap(), fateIdStr)) { + throw new AccumuloException("Could not fail transaction: " + fateIdStr); + } + } + } + } else if (options.delete) { + adminLock = createAdminLock(context); + try (var fateStores = createFateStores(context, zk, adminLock)) { + for (String fateIdStr : options.fateIdList) { + if (!admin.prepDelete(fateStores.getStoresMap(), fateIdStr)) { + throw new AccumuloException("Could not delete transaction: " + fateIdStr); + } + admin.deleteLocks(zk, zTableLocksPath, fateIdStr); + } + } + } + + if (options.print) { + final Set fateIdFilter = new TreeSet<>(); + options.fateIdList.forEach(fateIdStr -> fateIdFilter.add(FateId.from(fateIdStr))); + EnumSet statusFilter = getCmdLineStatusFilters(options.states); + EnumSet typesFilter = + getCmdLineInstanceTypeFilters(options.instanceTypes); + readOnlyFateStores = createReadOnlyFateStores(context, zk); + admin.print(readOnlyFateStores, zk, zTableLocksPath, new Formatter(System.out), + fateIdFilter, statusFilter, typesFilter); + // print line break at the end + System.out.println(); + } + + if (options.summarize) { + if (readOnlyFateStores == null) { + readOnlyFateStores = createReadOnlyFateStores(context, zk); + } + summarizeFateTx(context, options, admin, readOnlyFateStores, zTableLocksPath); + } + } finally { + if (adminLock != null) { + adminLock.unlock(); + } + } + } + + private FateStores createFateStores(ServerContext context, ZooSession zk, ServiceLock adminLock) + throws InterruptedException, KeeperException { + var lockId = adminLock.getLockID(); + MetaFateStore mfs = new MetaFateStore<>(zk, lockId, null); + UserFateStore ufs = + new UserFateStore<>(context, SystemTables.FATE.tableName(), lockId, null); + return new FateStores(FateInstanceType.META, mfs, FateInstanceType.USER, ufs); + } + + private Map> createReadOnlyFateStores( + ServerContext context, ZooSession zk) throws InterruptedException, KeeperException { + ReadOnlyFateStore readOnlyMFS = new MetaFateStore<>(zk, null, null); + ReadOnlyFateStore readOnlyUFS = + new UserFateStore<>(context, SystemTables.FATE.tableName(), null, null); + return Map.of(FateInstanceType.META, readOnlyMFS, FateInstanceType.USER, readOnlyUFS); + } + + private ServiceLock createAdminLock(ServerContext context) throws InterruptedException { + var zk = context.getZooSession(); + UUID uuid = UUID.randomUUID(); + ServiceLockPath slp = context.getServerPaths().createAdminLockPath(); + ServiceLock adminLock = new ServiceLock(zk, slp, uuid); + AdminLockWatcher lw = new AdminLockWatcher(); + ServiceLockData.ServiceDescriptors descriptors = new ServiceLockData.ServiceDescriptors(); + descriptors.addService(new ServiceLockData.ServiceDescriptor(uuid, + ServiceLockData.ThriftService.NONE, "fake_admin_util_host", ResourceGroupId.DEFAULT)); + ServiceLockData sld = new ServiceLockData(descriptors); + String lockPath = slp.toString(); + String parentLockPath = lockPath.substring(0, lockPath.lastIndexOf("/")); + + try { + var zrw = zk.asReaderWriter(); + zrw.putPersistentData(parentLockPath, new byte[0], ZooUtil.NodeExistsPolicy.SKIP); + zrw.putPersistentData(lockPath, new byte[0], ZooUtil.NodeExistsPolicy.SKIP); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException("Error creating path in ZooKeeper", e); + } + + adminLock.lock(lw, sld); + lockAcquiredLatch.await(); + + return adminLock; + } + + private void validateFateUserInput(FateOpts cmd) { + if (cmd.cancel && cmd.fail || cmd.cancel && cmd.delete || cmd.fail && cmd.delete) { + throw new IllegalArgumentException( + "Can only perform one of the following at a time: cancel, fail or delete."); + } + if ((cmd.cancel || cmd.fail || cmd.delete) && cmd.fateIdList.isEmpty()) { + throw new IllegalArgumentException( + "At least one txId required when using cancel, fail or delete"); + } + } + + private void cancelSubmittedFateTxs(ServerContext context, List fateIdList) + throws AccumuloException { + for (String fateIdStr : fateIdList) { + FateId fateId = FateId.from(fateIdStr); + TFateId thriftFateId = fateId.toThrift(); + boolean cancelled = cancelFateOperation(context, thriftFateId); + if (cancelled) { + System.out.println("FaTE transaction " + fateId + " was cancelled or already completed."); + } else { + System.out + .println("FaTE transaction " + fateId + " was not cancelled, status may have changed."); + } + } + } + + private boolean cancelFateOperation(ClientContext context, TFateId thriftFateId) + throws AccumuloException { + FateService.Client client = null; + try { + client = ThriftClientTypes.FATE.getConnectionWithRetry(context); + return client.cancelFateOperation(TraceUtil.traceInfo(), context.rpcCreds(), thriftFateId); + } catch (Exception e) { + throw new AccumuloException(e); + } finally { + if (client != null) { + ThriftUtil.close(client, context); + } + } + } + + private void summarizeFateTx(ServerContext context, FateOpts cmd, AdminUtil admin, + Map> fateStores, ServiceLockPath tableLocksPath) + throws InterruptedException, AccumuloException, AccumuloSecurityException, KeeperException, + NamespaceNotFoundException { + + var zk = context.getZooSession(); + var transactions = admin.getStatus(fateStores, zk, tableLocksPath, null, null, null); + + // build id map - relies on unique ids for tables and namespaces + // used to look up the names of either table or namespace by id. + Map tidToNameMap = context.createTableIdToQualifiedNameMap(); + Map idsToNameMap = new HashMap<>(tidToNameMap.size() * 2); + tidToNameMap.forEach((tid, name) -> idsToNameMap.put(tid.canonical(), "t:" + name)); + context.namespaceOperations().namespaceIdMap().forEach((name, nsid) -> { + String prev = idsToNameMap.put(nsid, "ns:" + name); + if (prev != null) { + LOG.warn("duplicate id found for table / namespace id. table name: {}, namespace name: {}", + prev, name); + } + }); + + Set fateIdFilter = + cmd.fateIdList.stream().map(FateId::from).collect(Collectors.toSet()); + EnumSet statusFilter = getCmdLineStatusFilters(cmd.states); + EnumSet typesFilter = getCmdLineInstanceTypeFilters(cmd.instanceTypes); + + FateSummaryReport report = + new FateSummaryReport(idsToNameMap, fateIdFilter, statusFilter, typesFilter); + + // gather statistics + transactions.getTransactions().forEach(report::gatherTxnStatus); + if (cmd.printJson) { + printLines(Collections.singletonList(report.toJson())); + } else { + printLines(report.formatLines()); + } + } + + private void printLines(List lines) { + for (String nextLine : lines) { + if (nextLine == null) { + continue; + } + System.out.println(nextLine); + } + } + + /** + * If provided on the command line, get the TStatus values provided. + * + * @return a set of status filters, or null if none provided + */ + private EnumSet getCmdLineStatusFilters(List states) { + EnumSet statusFilter = null; + if (!states.isEmpty()) { + statusFilter = EnumSet.noneOf(ReadOnlyFateStore.TStatus.class); + for (String element : states) { + statusFilter.add(ReadOnlyFateStore.TStatus.valueOf(element)); + } + } + return statusFilter; + } + + /** + * If provided on the command line, get the FateInstanceType values provided. + * + * @return a set of fate instance types filters, or null if none provided + */ + private EnumSet getCmdLineInstanceTypeFilters(List instanceTypes) { + EnumSet typesFilter = null; + if (!instanceTypes.isEmpty()) { + typesFilter = EnumSet.noneOf(FateInstanceType.class); + for (String instanceType : instanceTypes) { + typesFilter.add(FateInstanceType.valueOf(instanceType)); + } + } + return typesFilter; + } + + /** + * Finds tablets that point to fate operations that do not exists or are complete. + * + * @param tablets the tablets to inspect + * @param tabletLookup a function that can lookup a tablets latest metadata + * @param activePredicate a predicate that can determine if a fate id is currently active + * @param danglingConsumer a consumer that tablets with inactive fate ids will be sent to + */ + static void findDanglingFateOperations(Iterable tablets, + Function,Map> tabletLookup, + Predicate activePredicate, BiConsumer> danglingConsumer, + int bufferSize) { + + ArrayList fateIds = new ArrayList<>(); + Map> candidates = new HashMap<>(); + for (TabletMetadata tablet : tablets) { + fateIds.clear(); + getAllFateIds(tablet, fateIds::add); + fateIds.removeIf(activePredicate); + if (!fateIds.isEmpty()) { + candidates.put(tablet.getExtent(), new HashSet<>(fateIds)); + if (candidates.size() > bufferSize) { + processCandidates(candidates, tabletLookup, danglingConsumer); + candidates.clear(); + } + } + } + + processCandidates(candidates, tabletLookup, danglingConsumer); + } + + private static void processCandidates(Map> candidates, + Function,Map> tabletLookup, + BiConsumer> danglingConsumer) { + // Perform a 2nd check of the tablet to avoid race conditions like the following. + // 1. THREAD 1 : TabletMetadata is read and points to active fate operation + // 2. THREAD 2 : The fate operation is deleted from the tablet + // 3. THREAD 2 : The fate operation completes + // 4. THREAD 1 : Checks if the fate operation read in step 1 is active and finds it is not + + Map currentTablets = tabletLookup.apply(candidates.keySet()); + HashSet currentFateIds = new HashSet<>(); + candidates.forEach((extent, fateIds) -> { + var currentTablet = currentTablets.get(extent); + if (currentTablet != null) { + currentFateIds.clear(); + getAllFateIds(currentTablet, currentFateIds::add); + // Only keep fate ids that are still present in the tablet. Any new fate ids in + // currentFateIds that were not seen on the first pass are not considered here. To check + // those new ones, the entire two-step process would need to be rerun. + fateIds.retainAll(currentFateIds); + + if (!fateIds.isEmpty()) { + // the fateIds in this set were found to be inactive and still exist in the tablet + // metadata after being found inactive + danglingConsumer.accept(extent, fateIds); + } + } // else the tablet no longer exist so nothing to report + }); + } + + /** + * Extracts all fate ids that a tablet points to from any field. + */ + private static void getAllFateIds(TabletMetadata tabletMetadata, + Consumer fateIdConsumer) { + tabletMetadata.getLoaded().values().forEach(fateIdConsumer); + if (tabletMetadata.getSelectedFiles() != null) { + fateIdConsumer.accept(tabletMetadata.getSelectedFiles().getFateId()); + } + if (tabletMetadata.getOperationId() != null) { + fateIdConsumer.accept(tabletMetadata.getOperationId().getFateId()); + } + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListInstances.java similarity index 78% rename from server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListInstances.java index 6be707062bf..d695dc445eb 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListInstances.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; @@ -29,9 +29,7 @@ import java.util.TreeSet; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.cli.Help; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; import org.apache.accumulo.core.lock.ServiceLock; @@ -39,12 +37,19 @@ import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.lock.ServiceLockPaths; import org.apache.accumulo.core.zookeeper.ZooSession; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.ListInstances.Opts; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; -public class ListInstances { +@AutoService(KeywordExecutable.class) +public class ListInstances extends ServerKeywordExecutable { private static final Logger log = LoggerFactory.getLogger(ListInstances.class); @@ -54,7 +59,7 @@ public class ListInstances { private static final int ZOOKEEPER_TIMER_MILLIS = 30_000; - static class Opts extends Help { + static class Opts extends ServerUtilOpts { @Parameter(names = "--print-errors", description = "display errors while listing instances") boolean printErrors = false; @Parameter(names = "--print-all", @@ -64,25 +69,38 @@ static class Opts extends Help { String keepers = null; } - static Opts opts = new Opts(); - static int errors = 0; + int errors = 0; - public static void main(String[] args) throws InterruptedException { - opts.parseArgs(ListInstances.class.getName(), args); + public ListInstances() { + super(new Opts()); + } - if (opts.keepers == null) { - var siteConfig = SiteConfiguration.auto(); - opts.keepers = siteConfig.get(Property.INSTANCE_ZK_HOST); - } + @Override + public String keyword() { + return "list-instances"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "List Accumulo instances in zookeeper"; + } - String keepers = opts.keepers; - boolean printAll = opts.printAll; - boolean printErrors = opts.printErrors; + @Override + public void execute(JCommander cl, Opts options) throws Exception { + if (options.keepers == null) { + options.keepers = + options.getServerContext().getConfiguration().get(Property.INSTANCE_ZK_HOST); + } - listInstances(keepers, printAll, printErrors); + listInstances(options.keepers, options.printAll, options.printErrors); } - static synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) + synchronized void listInstances(String keepers, boolean printAll, boolean printErrors) throws InterruptedException { errors = 0; @@ -122,7 +140,7 @@ static synchronized void listInstances(String keepers, boolean printAll, boolean } } - private static class CharFiller implements Formattable { + private class CharFiller implements Formattable { char c; @@ -137,7 +155,7 @@ public void formatTo(Formatter formatter, int flags, int width, int precision) { } - private static void printHeader() { + private void printHeader() { System.out.printf(" %-" + NAME_WIDTH + "s| %-" + UUID_WIDTH + "s| %-" + MANAGER_WIDTH + "s%n", "Instance Name", "Instance ID", "Manager"); System.out.printf( @@ -146,7 +164,7 @@ private static void printHeader() { } - private static void printInstanceInfo(ZooSession zs, String instanceName, InstanceId iid, + private void printInstanceInfo(ZooSession zs, String instanceName, InstanceId iid, boolean printErrors) { String manager = getManager(zs, iid, printErrors); if (instanceName == null) { @@ -161,7 +179,7 @@ private static void printInstanceInfo(ZooSession zs, String instanceName, Instan "\"" + instanceName + "\"", iid, manager); } - private static String getManager(ZooSession zs, InstanceId iid, boolean printErrors) { + private String getManager(ZooSession zs, InstanceId iid, boolean printErrors) { if (iid == null) { return null; @@ -181,7 +199,7 @@ private static String getManager(ZooSession zs, InstanceId iid, boolean printErr } } - private static TreeMap getInstanceNames(ZooReader zk, boolean printErrors) { + private TreeMap getInstanceNames(ZooReader zk, boolean printErrors) { String instancesPath = Constants.ZROOT + Constants.ZINSTANCES; @@ -210,7 +228,7 @@ private static TreeMap getInstanceNames(ZooReader zk, boolean return tm; } - private static TreeSet getInstanceIDs(ZooReader zk, boolean printErrors) { + private TreeSet getInstanceIDs(ZooReader zk, boolean printErrors) { TreeSet ts = new TreeSet<>(); try { @@ -233,7 +251,7 @@ private static TreeSet getInstanceIDs(ZooReader zk, boolean printErr return ts; } - private static void handleException(Exception e, boolean printErrors) { + private void handleException(Exception e, boolean printErrors) { if (printErrors) { log.error("{}", e.getMessage(), e); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListVolumesUsed.java similarity index 74% rename from server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListVolumesUsed.java index 30b3d1d0edc..ad65253aef3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ListVolumesUsed.java @@ -16,29 +16,60 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import java.util.Iterator; import java.util.TreeSet; -import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.gc.GcCandidate; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.hadoop.fs.Path; -public class ListVolumesUsed { +import com.beust.jcommander.JCommander; +import com.google.auto.service.AutoService; - public static void main(String[] args) throws Exception { - listVolumes(new ServerContext(SiteConfiguration.auto())); +@AutoService(KeywordExecutable.class) +public class ListVolumesUsed extends ServerKeywordExecutable { + + public ListVolumesUsed() { + super(new ServerUtilOpts()); + } + + @Override + public String keyword() { + return "list-volumes"; } - private static String getTableURI(String rootTabletDir) { + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "list volumes currently in use"; + } + + @Override + public void execute(JCommander cl, ServerUtilOpts options) throws Exception { + ServerContext context = options.getServerContext(); + listTable(Ample.DataLevel.ROOT, context); + System.out.println(); + listTable(Ample.DataLevel.METADATA, context); + System.out.println(); + listTable(Ample.DataLevel.USER, context); + } + + private String getTableURI(String rootTabletDir) { Path ret = FileType.TABLE.getVolume(new Path(rootTabletDir)); if (ret == null) { return "RELATIVE"; @@ -46,7 +77,7 @@ private static String getTableURI(String rootTabletDir) { return ret.toString(); } - private static String getLogURI(String logEntry) { + private String getLogURI(String logEntry) { Path ret = FileType.WAL.getVolume(new Path(logEntry)); if (ret == null) { return "RELATIVE"; @@ -54,11 +85,11 @@ private static String getLogURI(String logEntry) { return ret.toString(); } - private static void getLogURIs(TreeSet volumes, LogEntry logEntry) { + private void getLogURIs(TreeSet volumes, LogEntry logEntry) { volumes.add(getLogURI(logEntry.getPath())); } - private static void listTable(Ample.DataLevel level, ServerContext context) throws Exception { + private void listTable(Ample.DataLevel level, ServerContext context) throws Exception { System.out.println("Listing volumes referenced in " + level + " tablets section"); @@ -100,12 +131,4 @@ private static void listTable(Ample.DataLevel level, ServerContext context) thro } } - public static void listVolumes(ServerContext context) throws Exception { - listTable(Ample.DataLevel.ROOT, context); - System.out.println(); - listTable(Ample.DataLevel.METADATA, context); - System.out.println(); - listTable(Ample.DataLevel.USER, context); - } - } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Locks.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Locks.java new file mode 100644 index 00000000000..fa93670cdfe --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/Locks.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class Locks extends ServerKeywordExecutable { + + public Locks() { + super(new ServerUtilOpts()); + } + + @Override + public String keyword() { + return "locks"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Prints server locks (Deprecated - use service-status and stop commands)"; + } + + @Override + public void execute(JCommander cl, ServerUtilOpts options) throws Exception { + System.out.println("'locks' command has been removed. Use 'service-status' command" + + " to list processes and 'stop -f' command to remove their locks."); + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/PingServer.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/PingServer.java new file mode 100644 index 00000000000..1422e0e9a58 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/PingServer.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.admin.InstanceOperations; +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.PingServer.PingCommandOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class PingServer extends ServerKeywordExecutable { + + static class PingCommandOpts extends ServerUtilOpts { + @Parameter(description = "{ ... }") + List args = new ArrayList<>(); + } + + public PingServer() { + super(new PingCommandOpts()); + } + + @Override + public String keyword() { + return "ping"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Ping tablet servers. If no arguments, pings all."; + } + + @Override + public void execute(JCommander cl, PingCommandOpts options) throws Exception { + + ServerContext context = options.getServerContext(); + InstanceOperations io = context.instanceOperations(); + + if (options.args.isEmpty()) { + io.getServers(ServerId.Type.TABLET_SERVER) + .forEach(t -> options.args.add(t.toHostPortString())); + } + + int unreachable = 0; + + for (String tserver : options.args) { + try { + io.ping(tserver); + System.out.println(tserver + " OK"); + } catch (AccumuloException ae) { + System.out.println(tserver + " FAILED (" + ae.getMessage() + ")"); + unreachable++; + } + } + + System.out.printf("\n%d of %d tablet servers unreachable\n\n", unreachable, + options.args.size()); + if (unreachable > 0) { + throw new IllegalStateException("Failed to ping some servers."); + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/RestoreZookeeper.java similarity index 64% rename from server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/RestoreZookeeper.java index 708837ea459..c7d8b325c20 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/RestoreZookeeper.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/RestoreZookeeper.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; @@ -30,17 +30,23 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; -import org.apache.accumulo.core.zookeeper.ZooSession; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.RestoreZookeeper.RestoreZooCommandOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.zookeeper.KeeperException; import org.xml.sax.Attributes; import org.xml.sax.helpers.DefaultHandler; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; -public class RestoreZookeeper { +@AutoService(KeywordExecutable.class) +public class RestoreZookeeper extends ServerKeywordExecutable { private static class Restore extends DefaultHandler { ZooReaderWriter zk = null; @@ -107,25 +113,48 @@ private void create(String path, String value, String encoding) { } } - @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", - justification = "code runs in same security context as user who provided input") - public static void execute(final AccumuloConfiguration conf, final String file, - final boolean overwrite) throws Exception { - try (var zk = new ZooSession(RestoreZookeeper.class.getSimpleName(), conf)) { - var zrw = zk.asReaderWriter(); + static class RestoreZooCommandOpts extends ServerUtilOpts { + @Parameter(names = "--overwrite") + boolean overwrite = false; - InputStream in = System.in; - if (file != null) { - in = Files.newInputStream(Path.of(file)); - } + @Parameter(names = "--file") + String file; + } + + public RestoreZookeeper() { + super(new RestoreZooCommandOpts()); + } + + @Override + public String keyword() { + return "restore-zookeeper"; + } - SAXParserFactory factory = SAXParserFactory.newInstance(); - // Prevent external entities by failing on any doctypes. We don't expect any doctypes, so this - // is a simple switch to remove any chance of external entities causing problems. - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - SAXParser parser = factory.newSAXParser(); - parser.parse(in, new Restore(zrw, overwrite)); - in.close(); + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Restore Zookeeper data from a file."; + } + + @Override + public void execute(JCommander cl, RestoreZooCommandOpts options) throws Exception { + ServerContext context = options.getServerContext(); + + InputStream in = System.in; + if (options.file != null) { + in = Files.newInputStream(Path.of(options.file)); } + + SAXParserFactory factory = SAXParserFactory.newInstance(); + // Prevent external entities by failing on any doctypes. We don't expect any doctypes, so this + // is a simple switch to remove any chance of external entities causing problems. + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + SAXParser parser = factory.newSAXParser(); + parser.parse(in, new Restore(context.getZooSession().asReaderWriter(), options.overwrite)); + in.close(); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ServiceStatus.java similarity index 87% rename from server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ServiceStatus.java index c43492d339f..ac01847eabe 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/ServiceStatus.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; @@ -35,25 +35,58 @@ import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.ServiceStatus.ServiceStatusCmdOpts; import org.apache.accumulo.server.util.serviceStatus.ServiceStatusReport; import org.apache.accumulo.server.util.serviceStatus.StatusSummary; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; import com.google.common.annotations.VisibleForTesting; -public class ServiceStatusCmd { +@AutoService(KeywordExecutable.class) +public class ServiceStatus extends ServerKeywordExecutable { - private static final Logger LOG = LoggerFactory.getLogger(ServiceStatusCmd.class); + private static final Logger LOG = LoggerFactory.getLogger(ServiceStatus.class); - public ServiceStatusCmd() {} + static class ServiceStatusCmdOpts extends ServerUtilOpts { - /** - * Read the service statuses from ZooKeeper, build the status report and then output the report to - * stdout. - */ - public void execute(final ServerContext context, final boolean json, final boolean showHosts) { + @Parameter(names = "--json", description = "provide output in json format") + boolean json = false; + + @Parameter(names = "--showHosts", + description = "provide a summary of service counts with host details") + boolean showHosts = false; + } + + public ServiceStatus() { + super(new ServiceStatusCmdOpts()); + } + + @Override + public String keyword() { + return "service-status"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Shows status of Accumulo server processes"; + } + + @Override + public void execute(JCommander cl, ServiceStatusCmdOpts options) throws Exception { + ServerContext context = options.getServerContext(); if (LOG.isTraceEnabled()) { LOG.trace("zooRoot: {}", ZooUtil.getRoot(context.getInstanceID())); @@ -68,9 +101,9 @@ public void execute(final ServerContext context, final boolean json, final boole services.put(ServiceStatusReport.ReportKey.COMPACTOR, getCompactorStatus(context)); services.put(ServiceStatusReport.ReportKey.GC, getGcStatus(context)); - ServiceStatusReport report = new ServiceStatusReport(services, showHosts); + ServiceStatusReport report = new ServiceStatusReport(services, options.showHosts); - if (json) { + if (options.json) { System.out.println(report.toJson()); } else { StringBuilder sb = new StringBuilder(8192); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopAll.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopAll.java new file mode 100644 index 00000000000..d2eeb27b7ed --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopAll.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.KeywordExecutable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.JCommander; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class StopAll extends ServerKeywordExecutable { + + private static final Logger LOG = LoggerFactory.getLogger(StopAll.class); + + // This only exists because it is called from StandaloneClusterControl, MiniAccumuloClusterControl + // and ITs + public static void main(String[] args) throws Exception { + new StopAll().execute(args); + } + + public StopAll() { + super(new ServerUtilOpts()); + } + + @Override + public String keyword() { + return "stop-all"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Stop all tablet servers and the manager."; + } + + @Override + public void execute(JCommander cl, ServerUtilOpts options) throws Exception { + ServerContext context = options.getServerContext(); + flushAll(context); + ThriftClientTypes.MANAGER.executeVoid(context, + client -> client.shutdown(TraceUtil.traceInfo(), context.rpcCreds(), true)); + } + + /** + * Flushing during shutdown is a performance optimization, it's not required. This method will + * attempt to initiate flushes of all tables and give up if it takes too long. + */ + private void flushAll(final ClientContext context) { + + final AtomicInteger flushesStarted = new AtomicInteger(0); + + Runnable flushTask = () -> { + try { + Set tables = context.tableOperations().list(); + for (String table : tables) { + if (table.equals(SystemTables.METADATA.tableName())) { + continue; + } + try { + context.tableOperations().flush(table, null, null, false); + flushesStarted.incrementAndGet(); + } catch (TableNotFoundException e) { + // ignore + } + } + } catch (Exception e) { + LOG.warn("Failed to initiate flush {}", e.getMessage()); + } + }; + + Thread flusher = new Thread(flushTask); + flusher.setDaemon(true); + flusher.start(); + + long start = System.currentTimeMillis(); + try { + flusher.join(3000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while waiting to join Flush thread", e); + } + + while (flusher.isAlive() && System.currentTimeMillis() - start < 15000) { + int flushCount = flushesStarted.get(); + try { + flusher.join(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while waiting to join Flush thread", e); + } + + if (flushCount == flushesStarted.get()) { + // no progress was made while waiting for join... maybe its stuck, stop waiting on it + break; + } + } + + flusher.interrupt(); + try { + flusher.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while waiting to join Flush thread", e); + } + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopManager.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopManager.java new file mode 100644 index 00000000000..19e05d5aeaf --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopManager.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class StopManager extends ServerKeywordExecutable { + + public StopManager() { + super(new ServerUtilOpts()); + } + + @Override + public String keyword() { + return "stop-manager"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Stop the manager."; + } + + @Override + public void execute(JCommander cl, ServerUtilOpts options) throws Exception { + ServerContext context = options.getServerContext(); + ThriftClientTypes.MANAGER.executeVoid(context, + client -> client.shutdown(TraceUtil.traceInfo(), context.rpcCreds(), false)); + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopServers.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopServers.java new file mode 100644 index 00000000000..fee585092ea --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/StopServers.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector; +import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate; +import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; +import org.apache.accumulo.core.process.thrift.ServerProcessService; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.core.zookeeper.ZooCache; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.ZooZap; +import org.apache.accumulo.server.util.adminCommand.StopServers.StopServersOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; +import com.google.common.net.HostAndPort; + +@AutoService(KeywordExecutable.class) +public class StopServers extends ServerKeywordExecutable { + + private static final Logger LOG = LoggerFactory.getLogger(StopServers.class); + + // This only exists because it is called from ITs + public static void main(String[] args) throws Exception { + new StopServers().execute(args); + } + + static class StopServersOpts extends ServerUtilOpts { + + @Parameter(names = {"-f", "--force"}, + description = "force the given server to stop immediately by removing its lock. Does not wait for any task the server is currently working.") + boolean force = false; + + @Parameter(description = " { ... }") + List args = new ArrayList<>(); + } + + public StopServers() { + super(new StopServersOpts()); + } + + @Override + public String keyword() { + return "stop-servers"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Stop the servers at the given addresses allowing them to complete current task but not start new task. Hostnames only are no longer supported; you must use . To Stop all services on a host, use 'accumulo admin service-status' to list all hosts and then pass them to this command."; + } + + @Override + public void execute(JCommander cl, StopServersOpts options) throws Exception { + + ServerContext context = options.getServerContext(); + List hostOnly = new ArrayList<>(); + Set hostAndPort = new TreeSet<>(); + + for (var server : options.args) { + if (server.contains(":")) { + hostAndPort.add(server); + } else { + hostOnly.add(server); + } + } + + if (!hostOnly.isEmpty()) { + // The old impl of this command with the old behavior + LOG.warn("Stopping by hostname is no longer supported\n\n" + + "please use instead.\n" + + "To stop all services on host, run 'accumulo admin serviceStatus' to list all host:port values and filter for that host and pass those to 'accumulo admin stop'"); + stopTabletServer(context, hostOnly, options.force); + } + + if (!hostAndPort.isEmpty()) { + // New behavior for this command when ports are present, supports more than just tservers. Is + // also async. + if (options.force) { + var zoo = context.getZooSession().asReaderWriter(); + + AddressSelector addresses = AddressSelector.matching(hostAndPort::contains); + List pathsToRemove = new ArrayList<>(); + pathsToRemove.addAll( + context.getServerPaths().getCompactor(ResourceGroupPredicate.ANY, addresses, false)); + pathsToRemove.addAll( + context.getServerPaths().getScanServer(ResourceGroupPredicate.ANY, addresses, false)); + pathsToRemove.addAll( + context.getServerPaths().getTabletServer(ResourceGroupPredicate.ANY, addresses, false)); + ZooZap.filterSingleton(context, context.getServerPaths().getManager(false), addresses) + .ifPresent(pathsToRemove::add); + ZooZap.filterSingleton(context, context.getServerPaths().getGarbageCollector(false), + addresses).ifPresent(pathsToRemove::add); + ZooZap.filterSingleton(context, context.getServerPaths().getMonitor(false), addresses) + .ifPresent(pathsToRemove::add); + + for (var path : pathsToRemove) { + List children = zoo.getChildren(path.toString()); + for (String child : children) { + LOG.trace("removing lock {}", path + "/" + child); + zoo.recursiveDelete(path + "/" + child, ZooUtil.NodeMissingPolicy.SKIP); + } + } + } else { + for (var server : hostAndPort) { + signalGracefulShutdown(context, HostAndPort.fromString(server)); + } + } + } + } + + // Visible for tests + public static void signalGracefulShutdown(final ClientContext context, HostAndPort hp) { + Objects.requireNonNull(hp, "address not set"); + ServerProcessService.Client client = null; + try { + client = ThriftClientTypes.SERVER_PROCESS.getServerProcessConnection(context, LOG, + hp.getHost(), hp.getPort()); + if (client == null) { + LOG.warn("Failed to initiate shutdown for {}", hp); + return; + } + client.gracefulShutdown(context.rpcCreds()); + LOG.info("Initiated shutdown for {}", hp); + } catch (TException e) { + LOG.warn("Failed to initiate shutdown for {}", hp, e); + } finally { + if (client != null) { + ThriftUtil.returnClient(client, context); + } + } + } + + /** + * Stops tablet servers by hostname + * + * @param context The server context + * @param servers LIst of hostnames (without ports) + * @param force Whether to force stop + * @deprecated Use servers with host:port format instead. To stop all services on a host use + * service status command to liat all services, then stop them with host:port format. + */ + @Deprecated(since = "4.0.0") + private void stopTabletServer(final ClientContext context, List servers, + final boolean force) throws AccumuloException, AccumuloSecurityException { + if (context.instanceOperations().getServers(ServerId.Type.MANAGER).isEmpty()) { + LOG.info("No managers running. Not attempting safe unload of tserver."); + return; + } + if (servers.isEmpty()) { + LOG.error("No tablet servers provided."); + return; + } + + final ZooCache zc = context.getZooCache(); + Set runningServers; + + for (String server : servers) { + runningServers = context.instanceOperations().getServers(ServerId.Type.TABLET_SERVER); + if (runningServers.size() == 1 && !force) { + LOG.info("Only 1 tablet server running. Not attempting shutdown of {}", server); + return; + } + for (int port : context.getConfiguration().getPort(Property.TSERV_CLIENTPORT)) { + HostAndPort address = AddressUtil.parseAddress(server, port); + final String finalServer = qualifyWithZooKeeperSessionId(context, zc, address.toString()); + LOG.info("Stopping server {}", finalServer); + ThriftClientTypes.MANAGER.executeVoid(context, client -> client + .shutdownTabletServer(TraceUtil.traceInfo(), context.rpcCreds(), finalServer, force)); + } + } + } + + /** + * Look up the TabletServers in ZooKeeper and try to find a sessionID for this server reference + * + * @param hostAndPort The host and port for a TabletServer + * @return The host and port with the session ID in square-brackets appended, or the original + * value. + */ + static String qualifyWithZooKeeperSessionId(ClientContext context, ZooCache zooCache, + String hostAndPort) { + var hpObj = HostAndPort.fromString(hostAndPort); + Set paths = context.getServerPaths() + .getTabletServer(ResourceGroupPredicate.ANY, AddressSelector.exact(hpObj), true); + if (paths.size() != 1) { + return hostAndPort; + } + long sessionId = ServiceLock.getSessionId(zooCache, paths.iterator().next()); + if (sessionId == 0) { + return hostAndPort; + } + return hostAndPort + "[" + Long.toHexString(sessionId) + "]"; + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/SystemCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/SystemCheck.java new file mode 100644 index 00000000000..88c15b989d8 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/SystemCheck.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckCommandOpts; +import org.apache.accumulo.server.util.checkCommand.CheckRunner; +import org.apache.accumulo.server.util.checkCommand.MetadataTableCheckRunner; +import org.apache.accumulo.server.util.checkCommand.RootMetadataCheckRunner; +import org.apache.accumulo.server.util.checkCommand.RootTableCheckRunner; +import org.apache.accumulo.server.util.checkCommand.ServerConfigCheckRunner; +import org.apache.accumulo.server.util.checkCommand.SystemConfigCheckRunner; +import org.apache.accumulo.server.util.checkCommand.SystemFilesCheckRunner; +import org.apache.accumulo.server.util.checkCommand.TableLocksCheckRunner; +import org.apache.accumulo.server.util.checkCommand.UserFilesCheckRunner; +import org.apache.accumulo.start.spi.KeywordExecutable; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; +import com.google.common.base.Preconditions; + +@AutoService(KeywordExecutable.class) +public class SystemCheck extends ServerKeywordExecutable { + + // This only exists because it is called from ITs + public static void main(String[] args) throws Exception { + new SystemCheck().execute(args); + } + + public static class CheckCommandOpts extends ServerUtilOpts { + @Parameter(names = "list", + description = "Lists the different checks that can be run, the description of each check, and the other check(s) each check depends on.") + boolean list; + + @Parameter(names = "run", + description = "Runs the provided check(s) (explicit list or regex pattern specified following '-p'), beginning with their dependencies, or all checks if none are provided.") + boolean run; + + @Parameter(names = {"-p", "--name_pattern"}, + description = "Runs all checks that match the provided regex pattern.") + String pattern; + + @Parameter(description = "[...]") + List checks; + + @Parameter(names = "--fixFiles", description = "Removes dangling file pointers. Used by the " + + "USER_FILES and SYSTEM_FILES checks.") + boolean fixFiles = false; + + /** + * This should be used to get the check runner instead of {@link Check#getCheckRunner()}. This + * exists so that its functionality can be changed for testing. + * + * @return the interface for running a check + */ + public CheckRunner getCheckRunner(Check check) { + return check.getCheckRunner(); + } + } + + public enum Check { + // Caution should be taken when changing or adding any new checks: order is important + SYSTEM_CONFIG(SystemConfigCheckRunner::new, "Validate the system config stored in ZooKeeper", + Collections.emptyList()), + SERVER_CONFIG(ServerConfigCheckRunner::new, "Validate the server configuration", + Collections.singletonList(SYSTEM_CONFIG)), + TABLE_LOCKS(TableLocksCheckRunner::new, + "Ensures that table and namespace locks are valid and are associated with a FATE op", + Collections.singletonList(SYSTEM_CONFIG)), + ROOT_METADATA(RootMetadataCheckRunner::new, + "Checks integrity of the root tablet metadata stored in ZooKeeper", + Collections.singletonList(SYSTEM_CONFIG)), + ROOT_TABLE(RootTableCheckRunner::new, + "Scans all the tablet metadata stored in the root table and checks integrity", + Collections.singletonList(ROOT_METADATA)), + METADATA_TABLE(MetadataTableCheckRunner::new, + "Scans all the tablet metadata stored in the metadata table and checks integrity", + Collections.singletonList(ROOT_TABLE)), + SYSTEM_FILES(SystemFilesCheckRunner::new, + "Checks that files in system tablet metadata exist in DFS", + Collections.singletonList(ROOT_TABLE)), + USER_FILES(UserFilesCheckRunner::new, "Checks that files in user tablet metadata exist in DFS", + Collections.singletonList(METADATA_TABLE)); + + private final Supplier checkRunner; + private final String description; + private final List dependencies; + + Check(Supplier checkRunner, String description, List dependencies) { + this.checkRunner = Objects.requireNonNull(checkRunner); + this.description = Objects.requireNonNull(description); + this.dependencies = Objects.requireNonNull(dependencies); + } + + /** + * This should not be called directly; use {@link CheckCommandOpts#getCheckRunner(Check)} + * instead + * + * + * @return the interface for running a check + */ + public CheckRunner getCheckRunner() { + return checkRunner.get(); + } + + /** + * @return the description of the check + */ + public String getDescription() { + return description; + } + + /** + * @return the list of other checks the check depends on + */ + public List getDependencies() { + return dependencies; + } + } + + public enum CheckStatus { + OK, FAILED, SKIPPED_DEPENDENCY_FAILED, FILTERED_OUT; + } + + public SystemCheck() { + super(new CheckCommandOpts()); + } + + // Visible for testing + public SystemCheck(T testOpts) { + super(testOpts); + } + + @Override + public String keyword() { + return "check"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Performs checks for problems in Accumulo."; + } + + @Override + public void execute(JCommander cl, CheckCommandOpts options) throws Exception { + + validateAndTransformCheckCommand(options); + + if (options.list) { + listChecks(); + } else if (options.run) { + ServerContext context = options.getServerContext(); + var givenChecks = options.checks.stream().map(name -> Check.valueOf(name.toUpperCase())) + .collect(Collectors.toList()); + executeRunCheckCommand(options, givenChecks, context); + } + } + + private void validateAndTransformCheckCommand(CheckCommandOpts cmd) { + Preconditions.checkArgument(cmd.list != cmd.run, "Must use either 'list' or 'run'"); + if (cmd.list) { + Preconditions.checkArgument(cmd.checks == null && cmd.pattern == null, + "'list' does not expect any further arguments"); + } else if (cmd.pattern != null) { + Preconditions.checkArgument(cmd.checks == null, "Expected one argument (the regex pattern)"); + List matchingChecks = new ArrayList<>(); + var pattern = Pattern.compile(cmd.pattern.toUpperCase()); + for (Check check : Check.values()) { + if (pattern.matcher(check.name()).matches()) { + matchingChecks.add(check.name()); + } + } + Preconditions.checkArgument(!matchingChecks.isEmpty(), + "No checks matched the given pattern: " + pattern.pattern()); + cmd.checks = matchingChecks; + } else { + if (cmd.checks == null) { + cmd.checks = + EnumSet.allOf(Check.class).stream().map(Enum::name).collect(Collectors.toList()); + } + } + } + + private void listChecks() { + System.out.println(); + System.out.printf("%-20s | %-90s | %-20s%n", "Check Name", "Description", "Depends on"); + System.out.println("-".repeat(130)); + for (Check check : Check.values()) { + System.out.printf("%-20s | %-90s | %-20s%n", check.name(), check.getDescription(), + check.getDependencies().stream().map(Check::name).collect(Collectors.joining(", "))); + } + System.out.println("-".repeat(130)); + System.out.println(); + } + + private void executeRunCheckCommand(CheckCommandOpts cmd, List givenChecks, + ServerContext context) throws Exception { + // Get all the checks in the order they are declared in the enum + final var allChecks = Check.values(); + final TreeMap checkStatus = new TreeMap<>(); + + for (Check check : allChecks) { + if (depsFailed(check, checkStatus)) { + checkStatus.put(check, CheckStatus.SKIPPED_DEPENDENCY_FAILED); + } else { + if (givenChecks.contains(check)) { + checkStatus.put(check, cmd.getCheckRunner(check).runCheck(context, cmd, cmd.fixFiles)); + } else { + checkStatus.put(check, CheckStatus.FILTERED_OUT); + } + } + } + + printChecksResults(checkStatus); + + if (checkStatus.values().stream().anyMatch(status -> status == CheckStatus.FAILED)) { + throw new IllegalStateException("One or more checks failed."); + } + } + + private boolean depsFailed(Check check, TreeMap checkStatus) { + return check.getDependencies().stream() + .anyMatch(dep -> checkStatus.get(dep) == CheckStatus.FAILED + || checkStatus.get(dep) == CheckStatus.SKIPPED_DEPENDENCY_FAILED); + } + + private void printChecksResults(TreeMap checkStatus) { + System.out.println(); + System.out.printf("%-20s | %-20s%n", "Check Name", "Status"); + System.out.println("-".repeat(50)); + for (Map.Entry entry : checkStatus.entrySet()) { + System.out.printf("%-20s | %-20s%n", entry.getKey().name(), entry.getValue().name()); + } + System.out.println("-".repeat(50)); + System.out.println(); + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/VerifyTabletAssignments.java similarity index 79% rename from server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java rename to server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/VerifyTabletAssignments.java index 36997c549f8..cacd24658d3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/adminCommand/VerifyTabletAssignments.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static org.apache.accumulo.core.util.threads.ThreadPoolNames.UTILITY_VERIFY_TABLET_ASSIGNMENTS; @@ -50,26 +50,62 @@ import org.apache.accumulo.core.util.HostAndPortComparator; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.cli.ServerUtilOpts; +import org.apache.accumulo.server.util.ServerKeywordExecutable; +import org.apache.accumulo.server.util.adminCommand.VerifyTabletAssignments.VerifyTabletAssignmentsOpts; +import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.hadoop.io.Text; import org.apache.thrift.TException; import org.apache.thrift.TServiceClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; import com.google.common.net.HostAndPort; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; -public class VerifyTabletAssignments { +@AutoService(KeywordExecutable.class) +public class VerifyTabletAssignments extends ServerKeywordExecutable { + private static final Logger log = LoggerFactory.getLogger(VerifyTabletAssignments.class); - public static void execute(ServerContext context, boolean verbose) throws Exception { + static class VerifyTabletAssignmentsOpts extends ServerUtilOpts { + @Parameter(names = {"-v", "--verbose"}, + description = "verbose mode (prints locations of tablets)") + boolean verbose = false; + } + + public VerifyTabletAssignments() { + super(new VerifyTabletAssignmentsOpts()); + } + + @Override + public String keyword() { + return "verify-tablet-assignments"; + } + + @Override + public UsageGroup usageGroup() { + return UsageGroup.ADMIN; + } + + @Override + public String description() { + return "Verify all Tablets are assigned to tablet servers"; + } + + @Override + public void execute(JCommander cl, VerifyTabletAssignmentsOpts options) throws Exception { + ServerContext context = options.getServerContext(); Span span = TraceUtil.startSpan(VerifyTabletAssignments.class, "main"); try (Scope scope = span.makeCurrent()) { try { for (String table : context.tableOperations().list()) { - checkTable(context, verbose, table, null); + checkTable(context, options.verbose, table, null); } } finally { span.end(); @@ -77,8 +113,8 @@ public static void execute(ServerContext context, boolean verbose) throws Except } } - private static void checkTable(final ClientContext context, final boolean verbose, - String tableName, HashSet check) throws InterruptedException { + private void checkTable(final ClientContext context, final boolean verbose, String tableName, + HashSet check) throws InterruptedException { if (check == null) { System.out.println("Checking table " + tableName); @@ -140,7 +176,7 @@ private static void checkTable(final ClientContext context, final boolean verbos } } - private static void checkFailures(HostAndPort server, HashSet failures, + private void checkFailures(HostAndPort server, HashSet failures, MultiScanResult scanResult) { for (TKeyExtent tke : scanResult.failures.keySet()) { KeyExtent ke = KeyExtent.fromThrift(tke); @@ -149,8 +185,8 @@ private static void checkFailures(HostAndPort server, HashSet failure } } - private static void checkTabletServer(ClientContext context, - Entry> entry, HashSet failures) throws TException { + private void checkTabletServer(ClientContext context, Entry> entry, + HashSet failures) throws TException { TabletScanClientService.Iface client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SCAN, entry.getKey(), context); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/CheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/CheckRunner.java index d816fb675b3..b5c51f408ef 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/CheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/CheckRunner.java @@ -20,7 +20,8 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,17 +35,17 @@ public interface CheckRunner { * @param opts server util opts. Only applicable for the checks on the root and metadata tables * @param fixFiles remove dangling file pointers. Only applicable for the checks on the system and * user files - * @return the {@link org.apache.accumulo.server.util.Admin.CheckCommand.CheckStatus} resulting - * from running the check + * @return the {@link org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus} + * resulting from running the check */ - Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception; + CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception; /** * * @return the check that this check runner runs */ - Admin.CheckCommand.Check getCheck(); + Check getCheck(); default void printRunning() { String running = "Running check " + getCheck(); @@ -53,7 +54,7 @@ default void printRunning() { log.trace("-".repeat(running.length())); } - default void printCompleted(Admin.CheckCommand.CheckStatus status) { + default void printCompleted(CheckStatus status) { String completed = "Check " + getCheck() + " completed with status " + status; log.trace("-".repeat(completed.length())); log.trace(completed); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataCheckRunner.java index 6d0b29bf113..042572cd2fd 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataCheckRunner.java @@ -40,7 +40,7 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.constraints.MetadataConstraints; import org.apache.accumulo.server.constraints.SystemEnvironment; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.apache.hadoop.io.Text; public interface MetadataCheckRunner extends CheckRunner { @@ -67,8 +67,8 @@ default String scanning() { * Ensures that the {@link #tableName()} table (either metadata or root table) has all columns * that are expected. For the root metadata, ensures that the expected "columns" exist in ZK. */ - default Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + default CheckStatus checkRequiredColumns(ServerContext context, CheckStatus status) + throws Exception { Set requiredColFQs; Set requiredColFams; boolean missingReqCol = false; @@ -99,7 +99,7 @@ default Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext contex if (!requiredColFQs.isEmpty() || !requiredColFams.isEmpty()) { log.warn("Tablet {} is missing required columns: col FQs: {}, col fams: {} in the {}\n", entry.getKey().getRow(), requiredColFQs, requiredColFams, scanning()); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; missingReqCol = true; } } @@ -115,9 +115,8 @@ default Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext contex * Ensures each column in the root or metadata table (or in ZK for the root metadata) is valid - * no unexpected columns, and for the columns that are expected, ensures the values are valid */ - default Admin.CheckCommand.CheckStatus checkColumns(ServerContext context, - Iterator> iter, - Admin.CheckCommand.CheckStatus status) { + default CheckStatus checkColumns(ServerContext context, + Iterator> iter, CheckStatus status) { boolean invalidCol = false; MetadataConstraints mc = new MetadataConstraints(); @@ -133,7 +132,7 @@ default Admin.CheckCommand.CheckStatus checkColumns(ServerContext context, var violations = mc.check(new ConstraintEnv(context), m); if (!violations.isEmpty()) { violations.forEach(violationCode -> log.warn(mc.getViolationDescription(violationCode))); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; invalidCol = true; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataTableCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataTableCheckRunner.java index 7ca81bd254d..0a5c321dc29 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataTableCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/MetadataTableCheckRunner.java @@ -27,13 +27,14 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.CheckForMetadataProblems; import org.apache.accumulo.server.util.FindOfflineTablets; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.apache.hadoop.io.Text; public class MetadataTableCheckRunner implements MetadataCheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.METADATA_TABLE; + private static final Check check = Check.METADATA_TABLE; @Override public String tableName() { @@ -51,14 +52,14 @@ public Set requiredColFams() { } @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Looking for offline tablets **********"); if (FindOfflineTablets.findOffline(context, null, true, true, log::trace, log::warn) != 0) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { log.trace("All good... No offline tablets found"); } @@ -66,7 +67,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil log.trace("********** Checking some references **********"); if (CheckForMetadataProblems.checkMetadataAndRootTableEntries(tableName(), opts, log::trace, log::warn)) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } log.trace("********** Looking for missing columns **********"); @@ -83,7 +84,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootMetadataCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootMetadataCheckRunner.java index d5d65d00e26..48ed5607aa9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootMetadataCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootMetadataCheckRunner.java @@ -30,12 +30,13 @@ import org.apache.accumulo.core.util.ColumnFQ; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.FindOfflineTablets; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.apache.hadoop.io.Text; public class RootMetadataCheckRunner implements MetadataCheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.ROOT_METADATA; + private static final Check check = Check.ROOT_METADATA; @Override public String tableName() { @@ -53,15 +54,15 @@ public String scanning() { } @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Looking for offline tablets **********"); if (FindOfflineTablets.findOffline(context, SystemTables.ROOT.tableName(), false, true, log::trace, log::warn) != 0) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { log.trace("All good... No offline tablets found"); } @@ -80,8 +81,8 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + public CheckStatus checkRequiredColumns(ServerContext context, CheckStatus status) + throws Exception { final String json = new String(context.getZooSession().asReader().getData(RootTable.ZROOT_TABLET), UTF_8); final var rtm = new RootTabletMetadata(json); @@ -101,14 +102,14 @@ public Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext context }); if (rowsSeen.size() != 1) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; log.warn("Did not see one tablet for the root table!"); } else { if (!requiredColFQs.isEmpty() || !requiredColFams.isEmpty()) { log.warn("Tablet {} is missing required columns: col FQs: {}, col fams: {} in the {}\n", rowsSeen.stream().findFirst().orElseThrow(), requiredColFQs, requiredColFams, scanning()); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { log.trace("...The {} contains all required columns for the root tablet\n", scanning()); } @@ -118,7 +119,7 @@ public Admin.CheckCommand.CheckStatus checkRequiredColumns(ServerContext context } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootTableCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootTableCheckRunner.java index 6a834016ba7..4e5ba675885 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootTableCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/RootTableCheckRunner.java @@ -26,12 +26,13 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.CheckForMetadataProblems; import org.apache.accumulo.server.util.FindOfflineTablets; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; public class RootTableCheckRunner implements MetadataCheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.ROOT_TABLE; + private static final Check check = Check.ROOT_TABLE; @Override public String tableName() { @@ -44,15 +45,15 @@ public TableId tableId() { } @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Looking for offline tablets **********"); if (FindOfflineTablets.findOffline(context, SystemTables.METADATA.tableName(), true, false, log::trace, log::warn) != 0) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { log.trace("All good... No offline tablets found"); } @@ -60,7 +61,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil log.trace("********** Checking some references **********"); if (CheckForMetadataProblems.checkMetadataAndRootTableEntries(tableName(), opts, log::trace, log::warn)) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } log.trace("********** Looking for missing columns **********"); @@ -77,7 +78,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/ServerConfigCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/ServerConfigCheckRunner.java index 02003c8eb49..fcd8f0f1202 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/ServerConfigCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/ServerConfigCheckRunner.java @@ -25,15 +25,16 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; public class ServerConfigCheckRunner implements CheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.SERVER_CONFIG; + private static final Check check = Check.SERVER_CONFIG; @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Checking server configuration **********"); @@ -47,7 +48,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil var val = entry.getValue(); if (!Property.isValidProperty(key, val)) { log.warn("Invalid property (key={} val={}) found in the config", key, val); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } } @@ -71,7 +72,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil // it's valid if (confPropVal == null || confPropVal.isEmpty()) { log.warn("Required property {} is not set!", reqProp); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } } @@ -80,7 +81,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java index 798ecd9955b..efe6661d55a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java @@ -33,18 +33,19 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.log.WalStateManager; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.apache.hadoop.fs.Path; import com.google.common.collect.Sets; public class SystemConfigCheckRunner implements CheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.SYSTEM_CONFIG; + private static final Check check = Check.SYSTEM_CONFIG; @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Checking validity of some ZooKeeper nodes **********"); @@ -54,8 +55,8 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil return status; } - private static Admin.CheckCommand.CheckStatus checkZkNodes(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + private static CheckStatus checkZkNodes(ServerContext context, CheckStatus status) + throws Exception { status = checkZKLocks(context, status); status = checkZKTableNodes(context, status); status = checkZKWALsMetadata(context, status); @@ -63,8 +64,8 @@ private static Admin.CheckCommand.CheckStatus checkZkNodes(ServerContext context return status; } - private static Admin.CheckCommand.CheckStatus checkZKLocks(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + private static CheckStatus checkZKLocks(ServerContext context, CheckStatus status) + throws Exception { final ServerId.Type[] serverTypes = ServerId.Type.values(); log.trace("Checking ZooKeeper locks for Accumulo server processes..."); @@ -84,7 +85,7 @@ private static Admin.CheckCommand.CheckStatus checkZKLocks(ServerContext context if (servers.size() != 1) { log.warn("Expected 1 server to be found for {} but found {}", serverType, servers.size()); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { // no exception and 1 server found log.trace("Verified ZooKeeper lock for {}", servers); @@ -96,7 +97,7 @@ private static Admin.CheckCommand.CheckStatus checkZKLocks(ServerContext context log.debug("No {} appears to be running. This may or may not be expected", serverType); } else if (servers.size() > 1) { log.warn("More than 1 {} was found running. This is not expected", serverType); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { // no exception and 1 server found log.trace("Verified ZooKeeper lock for {}", servers); @@ -108,7 +109,7 @@ private static Admin.CheckCommand.CheckStatus checkZKLocks(ServerContext context // essential process(es) if (servers.isEmpty()) { log.warn("No {} appear to be running. This is not expected.", serverType); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } else { // no exception and >= 1 server found log.trace("Verified ZooKeeper lock(s) for {} servers", servers.size()); @@ -131,8 +132,8 @@ private static Admin.CheckCommand.CheckStatus checkZKLocks(ServerContext context return status; } - private static Admin.CheckCommand.CheckStatus checkZKTableNodes(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + private static CheckStatus checkZKTableNodes(ServerContext context, CheckStatus status) + throws Exception { log.trace("Checking ZooKeeper table nodes..."); final var zrw = context.getZooSession().asReaderWriter(); @@ -147,22 +148,22 @@ private static Admin.CheckCommand.CheckStatus checkZKTableNodes(ServerContext co log.warn( "Missing essential Accumulo table. One or more of {} are missing from the tables found {}", systemTableNameToId, tableNameToId); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } for (var nameToId : tableNameToId.entrySet()) { var tablePath = Constants.ZTABLES + "/" + nameToId.getValue(); // expect the table path to exist and some data to exist if (!zrw.exists(tablePath) || zrw.getChildren(tablePath).isEmpty()) { log.warn("Failed to find table ({}) info at expected path {}", nameToId, tablePath); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } } return status; } - private static Admin.CheckCommand.CheckStatus checkZKWALsMetadata(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { + private static CheckStatus checkZKWALsMetadata(ServerContext context, CheckStatus status) + throws Exception { final var zrw = context.getZooSession().asReaderWriter(); log.trace("Checking that WAL metadata in ZooKeeper is valid..."); @@ -189,7 +190,7 @@ private static Admin.CheckCommand.CheckStatus checkZKWALsMetadata(ServerContext if (!actualMissing.isEmpty()) { log.warn("WAL metadata for tserver {} references a WAL that does not exist : {}", instanceAndMissingWals.getKey(), actualMissing); - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } } @@ -233,7 +234,7 @@ private static Admin.CheckCommand.CheckStatus checkZKWALsMetadata(ServerContext } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemFilesCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemFilesCheckRunner.java index 26bc49054c4..5fe4b85ec4c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemFilesCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemFilesCheckRunner.java @@ -21,22 +21,23 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.RemoveEntriesForMissingFiles; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; public class SystemFilesCheckRunner implements CheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.SYSTEM_FILES; + private static final Check check = Check.SYSTEM_FILES; @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Looking for missing system files **********"); if (RemoveEntriesForMissingFiles.checkTable(context, SystemTables.METADATA.tableName(), fixFiles, log::trace, log::warn) != 0) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } printCompleted(status); @@ -44,7 +45,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/TableLocksCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/TableLocksCheckRunner.java index beb09153301..c0e477b603e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/TableLocksCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/TableLocksCheckRunner.java @@ -29,15 +29,16 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; public class TableLocksCheckRunner implements CheckRunner { - private static final Admin.CheckCommand.Check check = Admin.CheckCommand.Check.TABLE_LOCKS; + private static final Check check = Check.TABLE_LOCKS; @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Checking some references **********"); @@ -48,17 +49,18 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } - private static Admin.CheckCommand.CheckStatus checkTableLocks(ServerContext context, - Admin.CheckCommand.CheckStatus status) throws Exception { - final AdminUtil admin = new AdminUtil<>(); + private static CheckStatus checkTableLocks(ServerContext context, CheckStatus status) + throws Exception { + final AdminUtil admin = new AdminUtil<>(); final var zTableLocksPath = context.getServerPaths().createTableLocksPath(); final var zk = context.getZooSession(); - try (final MetaFateStore mfs = new MetaFateStore<>(zk, null, null); final UserFateStore< - Admin> ufs = new UserFateStore<>(context, SystemTables.FATE.tableName(), null, null)) { + try (final MetaFateStore mfs = new MetaFateStore<>(zk, null, null); + final UserFateStore ufs = + new UserFateStore<>(context, SystemTables.FATE.tableName(), null, null)) { log.trace("Ensuring table and namespace locks are valid..."); @@ -72,7 +74,7 @@ private static Admin.CheckCommand.CheckStatus checkTableLocks(ServerContext cont lockedIds.removeAll(tableIds); lockedIds.removeAll(namespaceIds); if (!lockedIds.isEmpty()) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; log.warn("...Some table and namespace locks are INVALID (the table/namespace DNE): " + lockedIds); } else { @@ -90,7 +92,7 @@ private static Admin.CheckCommand.CheckStatus checkTableLocks(ServerContext cont zTableLocksPath, null, null, null); if (!fateStatus.getDanglingHeldLocks().isEmpty() || !fateStatus.getDanglingWaitingLocks().isEmpty()) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; log.warn("The following locks did not have an associated FATE operation\n"); for (Map.Entry> entry : fateStatus.getDanglingHeldLocks() .entrySet()) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/UserFilesCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/UserFilesCheckRunner.java index 54e7b9a14ce..5e54d789204 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/UserFilesCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/UserFilesCheckRunner.java @@ -18,21 +18,20 @@ */ package org.apache.accumulo.server.util.checkCommand; -import static org.apache.accumulo.server.util.Admin.CheckCommand.Check; - import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.RemoveEntriesForMissingFiles; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; public class UserFilesCheckRunner implements CheckRunner { private static final Check check = Check.USER_FILES; @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = Admin.CheckCommand.CheckStatus.OK; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = CheckStatus.OK; printRunning(); log.trace("********** Looking for missing user files **********"); @@ -42,7 +41,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil log.trace("Checking table {} ({}) for missing files\n", tableName, tableId); if (RemoveEntriesForMissingFiles.checkTable(context, tableName, fixFiles, log::trace, log::warn) != 0) { - status = Admin.CheckCommand.CheckStatus.FAILED; + status = CheckStatus.FAILED; } } } @@ -52,7 +51,7 @@ public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtil } @Override - public Admin.CheckCommand.Check getCheck() { + public Check getCheck() { return check; } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java deleted file mode 100644 index 831e7286bb1..00000000000 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.util; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; - -import org.junit.jupiter.api.Test; - -public class AdminCommandsTest { - @Test - public void testStopCommand() { - Admin.StopCommand cmd = new Admin.StopCommand(); - assertEquals(0, cmd.args.size()); - } - - @Test - public void testPingCommand() { - Admin.PingCommand cmd = new Admin.PingCommand(); - assertEquals(0, cmd.args.size()); - } - - @Test - public void testStopManagerCommand() { - new Admin.StopManagerCommand(); - } - - @Test - public void testStopAllCommand() { - new Admin.StopAllCommand(); - } - - @Test - public void testListInstancesCommand() { - Admin.ListInstancesCommand cmd = new Admin.ListInstancesCommand(); - assertFalse(cmd.printErrors); - assertFalse(cmd.printAll); - } - - @Test - public void testVolumesCommand() { - Admin.VolumesCommand cmd = new Admin.VolumesCommand(); - assertFalse(cmd.printErrors); - } - - @Test - public void testDumpConfigCommand() { - Admin.DumpConfigCommand cmd = new Admin.DumpConfigCommand(); - assertEquals(0, cmd.tables.size()); - assertFalse(cmd.allConfiguration); - assertFalse(cmd.systemConfiguration); - assertFalse(cmd.namespaceConfiguration); - assertFalse(cmd.users); - assertNull(cmd.directory); - } - - // not a command, but easy enough to include here - @Test - public void testStopOpts() { - Admin.StopCommand opts = new Admin.StopCommand(); - assertFalse(opts.force); - } -} diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/FateCmdTest.java similarity index 63% rename from server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java rename to server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/FateCmdTest.java index cc2ffaeeeb4..e5f514bd415 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/FateCmdTest.java @@ -16,26 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.getCurrentArguments; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -43,16 +34,10 @@ import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; -import org.apache.accumulo.core.lock.ServiceLockData; -import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; -import org.apache.accumulo.core.lock.ServiceLockPaths; import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.SelectedFiles; @@ -60,72 +45,10 @@ import org.apache.accumulo.core.metadata.schema.TabletOperationId; import org.apache.accumulo.core.metadata.schema.TabletOperationType; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.accumulo.core.zookeeper.ZcStat; -import org.apache.accumulo.core.zookeeper.ZooCache; import org.apache.hadoop.fs.Path; -import org.apache.zookeeper.KeeperException; -import org.easymock.EasyMock; import org.junit.jupiter.api.Test; -public class AdminTest { - - @Test - public void testQualifySessionId() throws KeeperException, InterruptedException { - ClientContext ctx = createMock(ClientContext.class); - ZooCache zc = createMock(ZooCache.class); - - String type = Constants.ZTSERVERS; - String group = type + "/" + Constants.DEFAULT_RESOURCE_GROUP_NAME; - String server = "localhost:12345"; - final long session = 123456789L; - ServiceLockData sld1 = new ServiceLockData(UUID.randomUUID(), server, ThriftService.TABLET_SCAN, - ResourceGroupId.DEFAULT); - - String serverPath = group + "/" + server; - String validZLockEphemeralNode = "zlock#" + UUID.randomUUID() + "#0000000000"; - expect(zc.getChildren(type)).andReturn(List.of(Constants.DEFAULT_RESOURCE_GROUP_NAME)) - .anyTimes(); - expect(zc.getChildren(group)).andReturn(List.of(server)).anyTimes(); - expect(zc.getChildren(serverPath)).andReturn(Collections.singletonList(validZLockEphemeralNode)) - .anyTimes(); - expect(zc.get(eq(serverPath + "/" + validZLockEphemeralNode), EasyMock.isA(ZcStat.class))) - .andReturn(sld1.serialize()).once(); - expect(zc.get(eq(serverPath + "/" + validZLockEphemeralNode), anyObject(ZcStat.class))) - .andAnswer(() -> { - ZcStat stat = (ZcStat) getCurrentArguments()[1]; - stat.setEphemeralOwner(session); - return new byte[0]; - }); - expect(ctx.getServerPaths()).andReturn(new ServiceLockPaths(zc)).anyTimes(); - replay(ctx, zc); - - assertEquals(server + "[" + Long.toHexString(session) + "]", - Admin.qualifyWithZooKeeperSessionId(ctx, zc, server)); - - verify(ctx, zc); - } - - @Test - public void testCannotQualifySessionId() throws KeeperException, InterruptedException { - ClientContext ctx = createMock(ClientContext.class); - ZooCache zc = createMock(ZooCache.class); - - String type = Constants.ZTSERVERS; - String group = type + "/" + Constants.DEFAULT_RESOURCE_GROUP_NAME; - String server = "localhost:12345"; - - String serverPath = group + "/" + server; - expect(zc.getChildren(type)).andReturn(List.of(Constants.DEFAULT_RESOURCE_GROUP_NAME)); - expect(zc.getChildren(serverPath)).andReturn(Collections.emptyList()); - expect(ctx.getServerPaths()).andReturn(new ServiceLockPaths(zc)).anyTimes(); - replay(ctx, zc); - - // A server that isn't in ZooKeeper. Can't qualify it, should return the original - assertEquals(server, Admin.qualifyWithZooKeeperSessionId(ctx, zc, server)); - - verify(ctx, zc); - } - +public class FateCmdTest { @Test public void testDanglingFate() { KeyExtent[] extents = new KeyExtent[10]; @@ -181,7 +104,7 @@ public void testDanglingFate() { }; // run test where every fate id is considered inactive - Admin.findDanglingFateOperations(tablets1.values(), tabletLookup, fateId -> false, found::put, + Fate.findDanglingFateOperations(tablets1.values(), tabletLookup, fateId -> false, found::put, 3); assertEquals(Map.of(tm1.getExtent(), Set.of(fateIds[0]), tm2.getExtent(), Set.of(fateIds[1]), tm3.getExtent(), Set.of(fateIds[2]), tm4.getExtent(), Set.of(fateIds[3]), tm5.getExtent(), @@ -191,7 +114,7 @@ public void testDanglingFate() { // run test where some of the fate ids are active Set active = Set.of(fateIds[0], fateIds[2], fateIds[4], fateIds[6]); found.clear(); - Admin.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, + Fate.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, 3); assertEquals(Map.of(tm2.getExtent(), Set.of(fateIds[1]), tm4.getExtent(), Set.of(fateIds[3]), tm5.getExtent(), Set.of(fateIds[5]), tm7.getExtent(), Set.of(fateIds[7])), found); @@ -206,7 +129,7 @@ public void testDanglingFate() { tablets2.put(tm4_1.getExtent(), tm4_1); tablets2.remove(tm5.getExtent()); found.clear(); - Admin.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, + Fate.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, 3); assertEquals(Map.of(tm7.getExtent(), Set.of(fateIds[7])), found); found.clear(); @@ -214,17 +137,18 @@ public void testDanglingFate() { // run a test where all are active on second look var tm7_1 = TabletMetadata.builder(tm7.getExtent()).putSelectedFiles(sf1).build(OPID, LOADED); tablets2.put(tm7_1.getExtent(), tm7_1); - Admin.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, + Fate.findDanglingFateOperations(tablets1.values(), tabletLookup, active::contains, found::put, 3); assertEquals(Map.of(), found); // run a test where all active on the first look active = Arrays.stream(fateIds).collect(Collectors.toSet()); found.clear(); - Admin.findDanglingFateOperations(tablets1.values(), le -> { + Fate.findDanglingFateOperations(tablets1.values(), le -> { assertTrue(le.isEmpty()); return Map.of(); }, active::contains, found::put, 3); assertEquals(Map.of(), found); } + } diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/ServiceStatusCmdTest.java similarity index 98% rename from server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java rename to server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/ServiceStatusCmdTest.java index 70c738ff7a5..0c5a856b761 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/ServiceStatusCmdTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.util; +package org.apache.accumulo.server.util.adminCommand; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.core.Constants.ZGC_LOCK; @@ -103,7 +103,7 @@ public void testManagerHosts() throws Exception { .andReturn(lock3Data.getBytes(UTF_8)); replay(zooReader); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getManagerStatus(context); LOG.info("manager status data: {}", status); @@ -146,7 +146,7 @@ public void testMonitorHosts() throws Exception { replay(zooReader); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getMonitorStatus(context); LOG.info("monitor status data: {}", status); @@ -242,7 +242,7 @@ public void testTServerHosts() throws Exception { replay(zooCache); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getTServerStatus(context); LOG.info("tserver status data: {}", status); @@ -335,7 +335,7 @@ public void testScanServerHosts() throws Exception { replay(zooCache); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getScanServerStatus(context); assertEquals(4, status.getServiceCount()); @@ -395,7 +395,7 @@ public void testCompactorStatus() throws Exception { replay(zooCache); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getCompactorStatus(context); LOG.info("compactor group counts: {}", status); @@ -433,7 +433,7 @@ public void testGcHosts() throws Exception { replay(zooReader); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getGcStatus(context); LOG.info("gc server counts: {}", status); assertEquals(2, status.getResourceGroups().size()); @@ -480,7 +480,7 @@ public void zkNodeDeletedTest() throws Exception { replay(zooReader); - ServiceStatusCmd cmd = new ServiceStatusCmd(); + ServiceStatus cmd = new ServiceStatus(); StatusSummary status = cmd.getManagerStatus(context); LOG.info("manager status data: {}", status); @@ -497,7 +497,7 @@ public void zkNodeDeletedTest() throws Exception { @Test public void testServiceStatusCommandOpts() { replay(zooReader, zooCache); - Admin.ServiceStatusCmdOpts opts = new Admin.ServiceStatusCmdOpts(); + ServiceStatus.ServiceStatusCmdOpts opts = new ServiceStatus.ServiceStatusCmdOpts(); assertFalse(opts.json); assertFalse(opts.showHosts); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/StopServersTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/StopServersTest.java new file mode 100644 index 00000000000..84340bc00b6 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/util/adminCommand/StopServersTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.util.adminCommand; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.getCurrentArguments; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; +import org.apache.accumulo.core.lock.ServiceLockPaths; +import org.apache.accumulo.core.zookeeper.ZcStat; +import org.apache.accumulo.core.zookeeper.ZooCache; +import org.apache.zookeeper.KeeperException; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Test; + +public class StopServersTest { + + @Test + public void testQualifySessionId() throws KeeperException, InterruptedException { + ClientContext ctx = createMock(ClientContext.class); + ZooCache zc = createMock(ZooCache.class); + + String type = Constants.ZTSERVERS; + String group = type + "/" + Constants.DEFAULT_RESOURCE_GROUP_NAME; + String server = "localhost:12345"; + final long session = 123456789L; + ServiceLockData sld1 = new ServiceLockData(UUID.randomUUID(), server, ThriftService.TABLET_SCAN, + ResourceGroupId.DEFAULT); + + String serverPath = group + "/" + server; + String validZLockEphemeralNode = "zlock#" + UUID.randomUUID() + "#0000000000"; + expect(zc.getChildren(type)).andReturn(List.of(Constants.DEFAULT_RESOURCE_GROUP_NAME)) + .anyTimes(); + expect(zc.getChildren(group)).andReturn(List.of(server)).anyTimes(); + expect(zc.getChildren(serverPath)).andReturn(Collections.singletonList(validZLockEphemeralNode)) + .anyTimes(); + expect(zc.get(eq(serverPath + "/" + validZLockEphemeralNode), EasyMock.isA(ZcStat.class))) + .andReturn(sld1.serialize()).once(); + expect(zc.get(eq(serverPath + "/" + validZLockEphemeralNode), anyObject(ZcStat.class))) + .andAnswer(() -> { + ZcStat stat = (ZcStat) getCurrentArguments()[1]; + stat.setEphemeralOwner(session); + return new byte[0]; + }); + expect(ctx.getServerPaths()).andReturn(new ServiceLockPaths(zc)).anyTimes(); + replay(ctx, zc); + + assertEquals(server + "[" + Long.toHexString(session) + "]", + StopServers.qualifyWithZooKeeperSessionId(ctx, zc, server)); + + verify(ctx, zc); + } + + @Test + public void testCannotQualifySessionId() throws KeeperException, InterruptedException { + ClientContext ctx = createMock(ClientContext.class); + ZooCache zc = createMock(ZooCache.class); + + String type = Constants.ZTSERVERS; + String group = type + "/" + Constants.DEFAULT_RESOURCE_GROUP_NAME; + String server = "localhost:12345"; + + String serverPath = group + "/" + server; + expect(zc.getChildren(type)).andReturn(List.of(Constants.DEFAULT_RESOURCE_GROUP_NAME)); + expect(zc.getChildren(serverPath)).andReturn(Collections.emptyList()); + expect(ctx.getServerPaths()).andReturn(new ServiceLockPaths(zc)).anyTimes(); + replay(ctx, zc); + + // A server that isn't in ZooKeeper. Can't qualify it, should return the original + assertEquals(server, StopServers.qualifyWithZooKeeperSessionId(ctx, zc, server)); + + verify(ctx, zc); + } + +} diff --git a/start/src/main/java/org/apache/accumulo/start/spi/KeywordExecutable.java b/start/src/main/java/org/apache/accumulo/start/spi/KeywordExecutable.java index 3866149fd07..04e2c80169d 100644 --- a/start/src/main/java/org/apache/accumulo/start/spi/KeywordExecutable.java +++ b/start/src/main/java/org/apache/accumulo/start/spi/KeywordExecutable.java @@ -45,7 +45,7 @@ public interface KeywordExecutable { enum UsageGroup { - COMPACTION, CORE, PROCESS, OTHER + ADMIN, COMPACTION, CORE, PROCESS, OTHER } /** diff --git a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java index d5c42a4657b..eb803f7f222 100644 --- a/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java +++ b/test/src/main/java/org/apache/accumulo/test/DumpConfigIT.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.DumpConfig; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.test.functional.FunctionalTestUtils; import org.apache.hadoop.conf.Configuration; @@ -75,7 +75,7 @@ public void test() throws Exception { } Path siteFileBackup = folder.resolve("accumulo.properties.bak"); assertFalse(Files.exists(siteFileBackup)); - assertEquals(0, exec(Admin.class, "dumpConfig", "-a", "-d", folder.toString()).waitFor()); + assertEquals(0, exec(DumpConfig.class, "-a", "-d", folder.toString()).waitFor()); assertTrue(Files.exists(siteFileBackup)); String site = FunctionalTestUtils.readAll(Files.newInputStream(siteFileBackup)); assertTrue(site.contains(Property.GENERAL_MAX_SCANNER_RETRY_PERIOD.getKey())); diff --git a/test/src/main/java/org/apache/accumulo/test/AdminCheckIT.java b/test/src/main/java/org/apache/accumulo/test/SystemCheckIT.java similarity index 78% rename from test/src/main/java/org/apache/accumulo/test/AdminCheckIT.java rename to test/src/main/java/org/apache/accumulo/test/SystemCheckIT.java index a51c8a264e2..cbf2ad982c3 100644 --- a/test/src/main/java/org/apache/accumulo/test/AdminCheckIT.java +++ b/test/src/main/java/org/apache/accumulo/test/SystemCheckIT.java @@ -56,21 +56,21 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.log.WalStateManager; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.SystemCheck; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.Check; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckCommandOpts; +import org.apache.accumulo.server.util.adminCommand.SystemCheck.CheckStatus; import org.apache.accumulo.server.util.checkCommand.CheckRunner; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.test.functional.ReadWriteIT; import org.apache.accumulo.test.functional.SlowIterator; import org.apache.hadoop.fs.Path; -import org.easymock.EasyMock; -import org.easymock.IAnswer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import com.beust.jcommander.JCommander; import com.google.common.collect.Sets; -public class AdminCheckIT extends ConfigurableMacBase { +public class SystemCheckIT extends ConfigurableMacBase { private static final PrintStream ORIGINAL_OUT = System.out; @AfterEach @@ -89,37 +89,35 @@ public void assertCorrectPostTestState() { public void testAdminCheckList() throws Exception { // verifies output of list command - var p = getCluster().exec(Admin.class, "check", "list"); + var p = getCluster().exec(SystemCheck.class, "list"); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); // Checks that the header is correct and that all checks are in the output assertTrue( out.contains("Check Name") && out.contains("Description") && out.contains("Depends on")); - List checksSeen = new ArrayList<>(); + List checksSeen = new ArrayList<>(); Arrays.stream(out.split("\\s+")).forEach(word -> { try { - checksSeen.add(Admin.CheckCommand.Check.valueOf(word)); + checksSeen.add(Check.valueOf(word)); } catch (IllegalArgumentException e) { // skip } }); - assertTrue(checksSeen.containsAll(List.of(Admin.CheckCommand.Check.values()))); + assertTrue(checksSeen.containsAll(List.of(Check.values()))); } @Test public void testAdminCheckListAndRunTogether() throws Exception { // Tries to execute list and run together; should not work - var p = getCluster().exec(Admin.class, "check", "list", "run"); + var p = getCluster().exec(SystemCheck.class, "list", "run"); assertNotEquals(0, p.getProcess().waitFor()); - p = getCluster().exec(Admin.class, "check", "run", "list"); + p = getCluster().exec(SystemCheck.class, "run", "list"); assertNotEquals(0, p.getProcess().waitFor()); - p = getCluster().exec(Admin.class, "check", "run", - Admin.CheckCommand.Check.SYSTEM_CONFIG.name(), "list"); + p = getCluster().exec(SystemCheck.class, "run", Check.SYSTEM_CONFIG.name(), "list"); assertNotEquals(0, p.getProcess().waitFor()); - p = getCluster().exec(Admin.class, "check", "list", - Admin.CheckCommand.Check.SYSTEM_CONFIG.name(), "run"); + p = getCluster().exec(SystemCheck.class, "list", Check.SYSTEM_CONFIG.name(), "run"); assertNotEquals(0, p.getProcess().waitFor()); } @@ -128,70 +126,69 @@ public void testAdminCheckListAndRunInvalidArgs() throws Exception { // tests providing invalid args to check // extra args to list - var p = getCluster().exec(Admin.class, "check", "list", "abc"); + var p = getCluster().exec(SystemCheck.class, "list", "abc"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("'list' does not expect any further arguments")); - p = getCluster().exec(Admin.class, "check", "list", - Admin.CheckCommand.Check.SYSTEM_CONFIG.name()); + p = getCluster().exec(SystemCheck.class, "list", Check.SYSTEM_CONFIG.name()); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("'list' does not expect any further arguments")); - p = getCluster().exec(Admin.class, "check", "list", "-p", "abc"); + p = getCluster().exec(SystemCheck.class, "list", "--name_pattern", "abc"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("'list' does not expect any further arguments")); // invalid check to run - p = getCluster().exec(Admin.class, "check", "run", "123"); + p = getCluster().exec(SystemCheck.class, "run", "123"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("IllegalArgumentException")); // no provided pattern - p = getCluster().exec(Admin.class, "check", "run", "-p"); + p = getCluster().exec(SystemCheck.class, "run", "--name_pattern"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("ParameterException")); // no checks match pattern - p = getCluster().exec(Admin.class, "check", "run", "-p", "abc"); + p = getCluster().exec(SystemCheck.class, "run", "--name_pattern", "abc"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("No checks matched the given pattern")); // invalid pattern - p = getCluster().exec(Admin.class, "check", "run", "-p", "[abc"); + p = getCluster().exec(SystemCheck.class, "run", "--name_pattern", "[abc"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("PatternSyntaxException")); // more than one arg provided to pattern - p = getCluster().exec(Admin.class, "check", "run", "-p", ".*files", ".*files"); + p = getCluster().exec(SystemCheck.class, "run", "--name_pattern", ".*files", ".*files"); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("Expected one argument (the regex pattern)")); // no list or run provided - p = getCluster().exec(Admin.class, "check"); + p = getCluster().exec(SystemCheck.class); assertNotEquals(0, p.getProcess().waitFor()); assertTrue(p.readStdOut().contains("Must use either 'list' or 'run'")); } @Test - public void testAdminCheckRunNoCheckFailures() { + public void testAdminCheckRunNoCheckFailures() throws Exception { // tests running the checks with none failing on run - Admin.CheckCommand.Check rootTableCheck = Admin.CheckCommand.Check.ROOT_TABLE; - Admin.CheckCommand.Check systemFilesCheck = Admin.CheckCommand.Check.SYSTEM_FILES; - Admin.CheckCommand.Check userFilesCheck = Admin.CheckCommand.Check.USER_FILES; + Check rootTableCheck = Check.ROOT_TABLE; + Check systemFilesCheck = Check.SYSTEM_FILES; + Check userFilesCheck = Check.USER_FILES; - boolean[] allChecksPass = new boolean[Admin.CheckCommand.Check.values().length]; + boolean[] allChecksPass = new boolean[Check.values().length]; Arrays.fill(allChecksPass, true); // no checks specified: should run all - String out1 = executeCheckCommand(new String[] {"check", "run"}, allChecksPass); + String out1 = executeCheckCommand(new String[] {"run"}, allChecksPass); // all checks specified: should run all - String[] allChecksArgs = new String[Admin.CheckCommand.Check.values().length + 2]; - allChecksArgs[0] = "check"; - allChecksArgs[1] = "run"; - for (int i = 2; i < allChecksArgs.length; i++) { - allChecksArgs[i] = Admin.CheckCommand.Check.values()[i - 2].name(); + String[] allChecksArgs = new String[Check.values().length + 1]; + allChecksArgs[0] = "run"; + for (int i = 1; i < allChecksArgs.length; i++) { + allChecksArgs[i] = Check.values()[i - 1].name(); } String out2 = executeCheckCommand(allChecksArgs, allChecksPass); // this pattern: should run all String out3 = - executeCheckCommand(new String[] {"check", "run", "-p", "[A-Z]+_[A-Z]+"}, allChecksPass); + executeCheckCommand(new String[] {"run", "--name_pattern", "[A-Z]+_[A-Z]+"}, allChecksPass); // run subset of checks - String out4 = executeCheckCommand(new String[] {"check", "run", rootTableCheck.name(), - systemFilesCheck.name(), userFilesCheck.name()}, allChecksPass); + String out4 = executeCheckCommand( + new String[] {"run", rootTableCheck.name(), systemFilesCheck.name(), userFilesCheck.name()}, + allChecksPass); // run same subset of checks but using a pattern to specify the checks (case shouldn't matter) - String out5 = executeCheckCommand(new String[] {"check", "run", "-p", "ROOT_TABLE|.*files"}, + String out5 = executeCheckCommand(new String[] {"run", "--name_pattern", "ROOT_TABLE|.*files"}, allChecksPass); String expRunAllRunOrder = @@ -241,7 +238,7 @@ public void testAdminCheckRunNoCheckFailures() { } @Test - public void testAdminCheckRunWithCheckFailures() { + public void testAdminCheckRunWithCheckFailures() throws Exception { // tests running checks with some failing boolean[] rootTableFails = new boolean[] {true, true, true, true, false, true, true, true}; @@ -251,20 +248,20 @@ public void testAdminCheckRunWithCheckFailures() { // run all checks with ROOT_TABLE failing: only SYSTEM_CONFIG and ROOT_METADATA should pass // the rest should be filtered out as skipped due to dependency failure - String out1 = executeCheckCommand(new String[] {"check", "run"}, rootTableFails); + String out1 = executeCheckCommand(new String[] {"run"}, rootTableFails); // run all checks with SYSTEM_CONFIG failing: only SYSTEM_CONFIG should run and fail // the rest should be filtered out as skipped due to dependency failure - String out2 = executeCheckCommand(new String[] {"check", "run"}, systemConfigFails); + String out2 = executeCheckCommand(new String[] {"run"}, systemConfigFails); // run subset of checks: SYSTEM_CONFIG, ROOT_TABLE, USER_FILES with USER_FILES and // METADATA_TABLE failing // should successfully run SYSTEM_CONFIG, ROOT_TABLE, fail to run USER_FILES and // filter out the rest - String out3 = executeCheckCommand( - new String[] {"check", "run", "SYSTEM_CONFIG", "ROOT_TABLE", "USER_FILES"}, - userFilesAndMetadataTableFails); + String out3 = + executeCheckCommand(new String[] {"run", "SYSTEM_CONFIG", "ROOT_TABLE", "USER_FILES"}, + userFilesAndMetadataTableFails); // run same subset but specified using pattern String out4 = executeCheckCommand( - new String[] {"check", "run", "-p", "SYSTEM_CONFIG|ROOT_TABLE|USER_FILES"}, + new String[] {"run", "--name_pattern", "SYSTEM_CONFIG|ROOT_TABLE|USER_FILES"}, userFilesAndMetadataTableFails); String expRunOrder1 = @@ -285,14 +282,11 @@ public void testAdminCheckRunWithCheckFailures() { assertTrue(out3.contains(expRunOrder3And4)); assertTrue(out4.contains(expRunOrder3And4)); - assertNoOtherChecksRan(out1, true, Admin.CheckCommand.Check.SYSTEM_CONFIG, - Admin.CheckCommand.Check.SERVER_CONFIG, Admin.CheckCommand.Check.TABLE_LOCKS, - Admin.CheckCommand.Check.ROOT_TABLE, Admin.CheckCommand.Check.ROOT_METADATA); - assertNoOtherChecksRan(out2, true, Admin.CheckCommand.Check.SYSTEM_CONFIG); - assertNoOtherChecksRan(out3, true, Admin.CheckCommand.Check.SYSTEM_CONFIG, - Admin.CheckCommand.Check.ROOT_TABLE, Admin.CheckCommand.Check.USER_FILES); - assertNoOtherChecksRan(out4, true, Admin.CheckCommand.Check.SYSTEM_CONFIG, - Admin.CheckCommand.Check.ROOT_TABLE, Admin.CheckCommand.Check.USER_FILES); + assertNoOtherChecksRan(out1, true, Check.SYSTEM_CONFIG, Check.SERVER_CONFIG, Check.TABLE_LOCKS, + Check.ROOT_TABLE, Check.ROOT_METADATA); + assertNoOtherChecksRan(out2, true, Check.SYSTEM_CONFIG); + assertNoOtherChecksRan(out3, true, Check.SYSTEM_CONFIG, Check.ROOT_TABLE, Check.USER_FILES); + assertNoOtherChecksRan(out4, true, Check.SYSTEM_CONFIG, Check.ROOT_TABLE, Check.USER_FILES); out1 = out1.replaceAll("\\s+", ""); out2 = out2.replaceAll("\\s+", ""); @@ -325,7 +319,7 @@ public void testAdminCheckRunWithCheckFailures() { @Test public void testTableLocksCheck() throws Exception { String table = getUniqueNames(1)[0]; - Admin.CheckCommand.Check tableLocksCheck = Admin.CheckCommand.Check.TABLE_LOCKS; + Check tableLocksCheck = Check.TABLE_LOCKS; try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { client.tableOperations().create(table); @@ -343,7 +337,7 @@ public void testTableLocksCheck() throws Exception { client.tableOperations().compact(table, slowCompaction); // test passing case - var p = getCluster().exec(Admin.class, "check", "run", tableLocksCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", tableLocksCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("locks are valid")); @@ -356,8 +350,8 @@ public void testTableLocksCheck() throws Exception { final var zrw = context.getZooSession().asReaderWriter(); final var path = new ServiceLockPaths(context.getZooCache()).createTableLocksPath(); zrw.putPersistentData(path.toString() + "/foo", new byte[0], ZooUtil.NodeExistsPolicy.FAIL); - p = getCluster().exec(Admin.class, "check", "run", tableLocksCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", tableLocksCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue( out.contains("Some table and namespace locks are INVALID (the table/namespace DNE)")); @@ -368,7 +362,7 @@ public void testTableLocksCheck() throws Exception { @Test public void testMetadataTableCheck() throws Exception { - Admin.CheckCommand.Check metaTableCheck = Admin.CheckCommand.Check.METADATA_TABLE; + Check metaTableCheck = Check.METADATA_TABLE; String table = getUniqueNames(1)[0]; try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { @@ -379,7 +373,7 @@ public void testMetadataTableCheck() throws Exception { } // test passing case - var p = getCluster().exec(Admin.class, "check", "run", metaTableCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", metaTableCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("Looking for offline tablets")); @@ -400,8 +394,8 @@ public void testMetadataTableCheck() throws Exception { MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier()); writer.addMutation(mut); } - p = getCluster().exec(Admin.class, "check", "run", metaTableCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", metaTableCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("Tablet " + tablet + " is missing required columns")); assertTrue(out.contains("Check METADATA_TABLE completed with status FAILED")); @@ -410,11 +404,11 @@ public void testMetadataTableCheck() throws Exception { @Test public void testRootTableCheck() throws Exception { - Admin.CheckCommand.Check rootTableCheck = Admin.CheckCommand.Check.ROOT_TABLE; + Check rootTableCheck = Check.ROOT_TABLE; // test passing case // no extra setup needed, just check the root table - var p = getCluster().exec(Admin.class, "check", "run", rootTableCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", rootTableCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("Looking for offline tablets")); @@ -435,8 +429,8 @@ public void testRootTableCheck() throws Exception { MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier()); writer.addMutation(mut); } - p = getCluster().exec(Admin.class, "check", "run", rootTableCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", rootTableCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("Tablet " + tablet + " is missing required columns")); assertTrue(out.contains("Check ROOT_TABLE completed with status FAILED")); @@ -445,11 +439,11 @@ public void testRootTableCheck() throws Exception { @Test public void testRootMetadataCheck() throws Exception { - Admin.CheckCommand.Check rootMetaCheck = Admin.CheckCommand.Check.ROOT_METADATA; + Check rootMetaCheck = Check.ROOT_METADATA; // test passing case // no extra setup needed, just check the root table metadata - var p = getCluster().exec(Admin.class, "check", "run", rootMetaCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", rootMetaCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("Looking for offline tablets")); @@ -472,8 +466,8 @@ public void testRootMetadataCheck() throws Exception { zrw.putPersistentData(RootTable.ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), ZooUtil.NodeExistsPolicy.OVERWRITE); - p = getCluster().exec(Admin.class, "check", "run", rootMetaCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", rootMetaCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("Tablet " + tablet + " is missing required columns")); assertTrue(out.contains("Check ROOT_METADATA completed with status FAILED")); @@ -482,11 +476,11 @@ public void testRootMetadataCheck() throws Exception { @Test public void testSystemFilesCheck() throws Exception { - Admin.CheckCommand.Check sysFilesCheck = Admin.CheckCommand.Check.SYSTEM_FILES; + Check sysFilesCheck = Check.SYSTEM_FILES; // test passing case // no extra setup needed, just run the check - var p = getCluster().exec(Admin.class, "check", "run", sysFilesCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", sysFilesCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(Pattern.compile("missing files: 0, total files: [1-9]+").matcher(out).find()); @@ -503,8 +497,8 @@ public void testSystemFilesCheck() throws Exception { path = new Path(StoredTabletFile.of(pathJsonData).getMetadataPath()); getCluster().getServerContext().getVolumeManager().delete(path); } - p = getCluster().exec(Admin.class, "check", "run", sysFilesCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", sysFilesCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("File " + path + " is missing")); assertTrue(Pattern.compile("missing files: 1, total files: [1-9]+").matcher(out).find()); @@ -514,7 +508,7 @@ public void testSystemFilesCheck() throws Exception { @Test public void testUserFilesCheck() throws Exception { - Admin.CheckCommand.Check userFilesCheck = Admin.CheckCommand.Check.USER_FILES; + Check userFilesCheck = Check.USER_FILES; try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { // test passing case @@ -524,7 +518,7 @@ public void testUserFilesCheck() throws Exception { ReadWriteIT.ingest(client, 10, 10, 10, 0, table); client.tableOperations().flush(table, null, null, true); - var p = getCluster().exec(Admin.class, "check", "run", userFilesCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", userFilesCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(Pattern.compile("missing files: 0, total files: [1-9]+").matcher(out).find()); @@ -541,8 +535,8 @@ public void testUserFilesCheck() throws Exception { path = new Path(StoredTabletFile.of(pathJsonData).getMetadataPath()); getCluster().getServerContext().getVolumeManager().delete(path); } - p = getCluster().exec(Admin.class, "check", "run", userFilesCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", userFilesCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("File " + path + " is missing")); assertTrue(Pattern.compile("missing files: 1, total files: [1-9]+").matcher(out).find()); @@ -553,10 +547,10 @@ public void testUserFilesCheck() throws Exception { @Test public void testSystemConfigCheck() throws Exception { - Admin.CheckCommand.Check sysConfCheck = Admin.CheckCommand.Check.SYSTEM_CONFIG; + Check sysConfCheck = Check.SYSTEM_CONFIG; // test passing case - var p = getCluster().exec(Admin.class, "check", "run", sysConfCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", sysConfCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("Checking ZooKeeper locks for Accumulo server processes")); @@ -572,8 +566,8 @@ public void testSystemConfigCheck() throws Exception { zrw.recursiveDelete(Constants.ZTABLES + "/" + SystemTables.METADATA.tableId(), ZooUtil.NodeMissingPolicy.FAIL); - p = getCluster().exec(Admin.class, "check", "run", sysConfCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + p = getCluster().exec(SystemCheck.class, "run", sysConfCheck.name()); + assertEquals(1, p.getProcess().waitFor()); out = p.readStdOut(); assertTrue(out.contains("Failed to find table (" + (Map.entry(SystemTables.METADATA.tableName(), SystemTables.METADATA.tableId())) + ")")); @@ -586,7 +580,7 @@ public void testSystemConfigCheck2() throws Exception { // test a failing case // delete a WAL in HDFS that is referenced in ZK - Admin.CheckCommand.Check sysConfCheck = Admin.CheckCommand.Check.SYSTEM_CONFIG; + Check sysConfCheck = Check.SYSTEM_CONFIG; var context = getCluster().getServerContext(); var zrw = context.getZooSession().asReaderWriter(); var rootWalsDir = WalStateManager.ZWALS; @@ -620,8 +614,8 @@ public void testSystemConfigCheck2() throws Exception { // delete from HDFS context.getVolumeManager().delete(wal.getSecond()); - var p = getCluster().exec(Admin.class, "check", "run", sysConfCheck.name()); - assertEquals(5, p.getProcess().waitFor()); + var p = getCluster().exec(SystemCheck.class, "run", sysConfCheck.name()); + assertEquals(1, p.getProcess().waitFor()); var out = p.readStdOut(); assertTrue(out.contains( "WAL metadata for tserver " + tServerInstance + " references a WAL that does not exist")); @@ -631,10 +625,10 @@ public void testSystemConfigCheck2() throws Exception { @Test public void testServerConfigCheck() throws Exception { - Admin.CheckCommand.Check servConfCheck = Admin.CheckCommand.Check.SERVER_CONFIG; + Check servConfCheck = Check.SERVER_CONFIG; // test passing case - var p = getCluster().exec(Admin.class, "check", "run", servConfCheck.name()); + var p = getCluster().exec(SystemCheck.class, "run", servConfCheck.name()); assertEquals(0, p.getProcess().waitFor()); String out = p.readStdOut(); assertTrue(out.contains("Checking server configuration")); @@ -646,50 +640,41 @@ public void testServerConfigCheck() throws Exception { // no simple way to test for a failure case } - private String executeCheckCommand(String[] checkCmdArgs, boolean[] checksPass) { + private String executeCheckCommand(String[] checkCmdArgs, boolean[] checksPass) throws Exception { String output; - Admin admin = createMockAdmin(checksPass); + SystemCheck check = createDummyCheckCommand(checksPass); try (ByteArrayOutputStream outStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(outStream)) { System.setOut(printStream); - admin.execute(checkCmdArgs); + try { + check.execute(checkCmdArgs); + } catch (IllegalStateException e) { + // This will happen if one of the commands fails. + // The output is checked by the calling command, + // so eat this exception. + } output = outStream.toString(); } catch (IOException e) { throw new UncheckedIOException(e); } finally { - EasyMock.verify(admin); System.setOut(ORIGINAL_OUT); } return output; } - private Admin createMockAdmin(boolean[] checksPass) { - // mocking admin.execute() to just execute "check" with our dummy check command - Admin admin = EasyMock.createMock(Admin.class); - admin.execute(EasyMock.anyObject(String[].class)); - EasyMock.expectLastCall().andAnswer((IAnswer) () -> { - String[] args = EasyMock.getCurrentArgument(0); - ServerUtilOpts opts = new ServerUtilOpts(); - JCommander cl = new JCommander(opts); - Admin.CheckCommand dummyCheckCommand = new DummyCheckCommand(checksPass); - cl.addCommand("check", dummyCheckCommand); - cl.parse(args); - Admin.executeCheckCommand(getCluster().getServerContext(), dummyCheckCommand, opts); - return null; - }); - EasyMock.replay(admin); - return admin; + private SystemCheck createDummyCheckCommand(boolean[] checksPass) throws Exception { + DummyCheckCommand opts = new DummyCheckCommand(checksPass); + SystemCheck check = new SystemCheck(opts); + return check; } /** * Asserts that no checks (other than those provided) ran. */ - private void assertNoOtherChecksRan(String out, boolean isDummyCheck, - Admin.CheckCommand.Check... checks) { - Set otherChecks = - Sets.difference(Set.of(Admin.CheckCommand.Check.values()), Set.of(checks)); + private void assertNoOtherChecksRan(String out, boolean isDummyCheck, Check... checks) { + Set otherChecks = Sets.difference(Set.of(Check.values()), Set.of(checks)); for (var check : otherChecks) { assertFalse( out.contains("Running " + (isDummyCheck ? "dummy " : "") + "check " + check.name())); @@ -704,10 +689,9 @@ public DummyCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, - boolean fixFiles) throws Exception { - Admin.CheckCommand.CheckStatus status = - passes ? Admin.CheckCommand.CheckStatus.OK : Admin.CheckCommand.CheckStatus.FAILED; + public CheckStatus runCheck(ServerContext context, ServerUtilOpts opts, boolean fixFiles) + throws Exception { + CheckStatus status = passes ? CheckStatus.OK : CheckStatus.FAILED; System.out.println("Running dummy check " + getCheck()); // no work to perform in the dummy check runner @@ -722,8 +706,8 @@ public DummySystemConfigCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.SYSTEM_CONFIG; + public Check getCheck() { + return Check.SYSTEM_CONFIG; } } @@ -733,8 +717,8 @@ public DummyServerConfigCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.SERVER_CONFIG; + public Check getCheck() { + return Check.SERVER_CONFIG; } } @@ -744,8 +728,8 @@ public DummyTableLocksCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.TABLE_LOCKS; + public Check getCheck() { + return Check.TABLE_LOCKS; } } @@ -755,8 +739,8 @@ public DummyRootMetadataCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.ROOT_METADATA; + public Check getCheck() { + return Check.ROOT_METADATA; } } @@ -766,8 +750,8 @@ public DummyRootTableCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.ROOT_TABLE; + public Check getCheck() { + return Check.ROOT_TABLE; } } @@ -777,8 +761,8 @@ public DummyMetadataTableCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.METADATA_TABLE; + public Check getCheck() { + return Check.METADATA_TABLE; } } @@ -788,8 +772,8 @@ public DummySystemFilesCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.SYSTEM_FILES; + public Check getCheck() { + return Check.SYSTEM_FILES; } } @@ -799,12 +783,12 @@ public DummyUserFilesCheckRunner(boolean passes) { } @Override - public Admin.CheckCommand.Check getCheck() { - return Admin.CheckCommand.Check.USER_FILES; + public Check getCheck() { + return Check.USER_FILES; } } - static class DummyCheckCommand extends Admin.CheckCommand { + class DummyCheckCommand extends CheckCommandOpts { final Map> checkRunners; public DummyCheckCommand(boolean[] checksPass) { @@ -824,6 +808,14 @@ public DummyCheckCommand(boolean[] checksPass) { this.checkRunners.put(Check.USER_FILES, () -> new DummyUserFilesCheckRunner(checksPass[7])); } + @Override + public synchronized ServerContext getServerContext() { + // Don't use the MiniAccumuloCluster's ServerContext + // because ServerKeywordExecutable will close it + // and cause errors during subsequent IT operations. + return new ServerContext(getCluster().getServerContext().getSiteConfiguration()); + } + @Override public CheckRunner getCheckRunner(Check check) { return checkRunners.get(check).get(); diff --git a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java index 8c873b4a778..1f8fff06be5 100644 --- a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java @@ -41,7 +41,7 @@ import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.miniclusterImpl.ProcessReference; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopAll; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.tserver.TabletServer; import org.apache.hadoop.conf.Configuration; @@ -117,7 +117,7 @@ public void testSerializedRecovery() throws Exception { try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) { scanner.forEach((k, v) -> {}); } - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); ts.getProcess().waitFor(); String result = ts.readStdOut(); log.info(result); diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java index f003dec6f22..32afdfe5605 100644 --- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java @@ -49,7 +49,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.init.Initialize; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopAll; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -122,7 +122,7 @@ private InstanceId verifyAndShutdownCluster(AccumuloClient c, String tableName) verifyVolumesUsed(c, tableName, false, v1, v2); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); cluster.stop(); return uuid; @@ -173,7 +173,7 @@ public void testRemoveVolumes() throws Exception { verifyVolumesUsed(client, tableNames[0], false, v1, v2); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); cluster.stop(); updateConfig(config -> config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v2.toString())); diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeITBase.java b/test/src/main/java/org/apache/accumulo/test/VolumeITBase.java index 80b69521983..abf7abed8a8 100644 --- a/test/src/main/java/org/apache/accumulo/test/VolumeITBase.java +++ b/test/src/main/java/org/apache/accumulo/test/VolumeITBase.java @@ -69,7 +69,7 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.security.SystemCredentials; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopAll; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.test.util.FileMetadataUtil; import org.apache.commons.configuration2.PropertiesConfiguration; @@ -268,7 +268,7 @@ protected void testReplaceVolume(AccumuloClient client, boolean cleanShutdown, } if (cleanShutdown) { - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); } cluster.stop(); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java index 0f62184103c..e0ff93e4a4f 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java @@ -73,7 +73,6 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.fateCommand.FateSummaryReport; import org.apache.accumulo.server.util.fateCommand.FateTxnDetails; import org.apache.accumulo.test.fate.MultipleStoresITBase.LatchTestEnv; @@ -128,7 +127,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte try { // validate blank report, no transactions have started - ProcessInfo p = getCluster().exec(Admin.class, "fate", "--summary", "-j"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -150,7 +150,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte List fateIdsStarted = List.of(fateId1.canonical(), fateId2.canonical()); // validate no filters - p = getCluster().exec(Admin.class, "fate", "--summary", "-j"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--summary", + "-j"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -171,8 +172,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte */ // validate filtering by both transactions - p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), fateId2.canonical(), - "--summary", "-j"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), fateId2.canonical(), "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -190,7 +191,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte validateFateDetails(report.getFateDetails(), 2, fateIdsStarted); // validate filtering by just one transaction - p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--summary", "-j"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -209,7 +211,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte // validate filtering by non-existent transaction FateId fakeFateId = FateId.from(store.type(), UUID.randomUUID()); - p = getCluster().exec(Admin.class, "fate", fakeFateId.canonical(), "--summary", "-j"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fakeFateId.canonical(), "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -231,7 +234,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte */ // validate status filter by including only FAILED transactions, should be none - p = getCluster().exec(Admin.class, "fate", "--summary", "-j", "-s", "FAILED"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--summary", + "-j", "-s", "FAILED"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -248,7 +252,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte validateFateDetails(report.getFateDetails(), 0, fateIdsStarted); // validate status filter by including only NEW transactions, should be 2 - p = getCluster().exec(Admin.class, "fate", "--summary", "-j", "-s", "NEW"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--summary", + "-j", "-s", "NEW"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -269,7 +274,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte */ // validate FateInstanceType filter by only including transactions with META filter - p = getCluster().exec(Admin.class, "fate", "--summary", "-j", "-t", "META"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--summary", + "-j", "-t", "META"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -290,7 +296,8 @@ protected void testFateSummaryCommand(FateStore store, ServerConte } // validate FateInstanceType filter by only including transactions with USER filter - p = getCluster().exec(Admin.class, "fate", "--summary", "-j", "-t", "USER"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--summary", + "-j", "-t", "USER"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -329,8 +336,9 @@ protected void testFateSummaryCommandPlainText(FateStore store, Se FateId fateId1 = fate.startTransaction(); FateId fateId2 = fate.startTransaction(); - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), - fateId2.canonical(), "--summary", "-s", "NEW", "-t", store.type().name()); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), fateId2.canonical(), "--summary", "-s", "NEW", "-t", + store.type().name()); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); assertTrue(result.contains("Status Filters: [NEW]")); @@ -356,7 +364,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext try { // validate no transactions - ProcessInfo p = getCluster().exec(Admin.class, "fate", "--print"); + ProcessInfo p = + getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); assertTrue(result.contains(" 0 transactions")); @@ -366,7 +375,7 @@ protected void testFatePrintCommand(FateStore store, ServerContext FateId fateId2 = fate.startTransaction(); // Get all transactions. Should be 2 FateIds with a NEW status - p = getCluster().exec(Admin.class, "fate", "--print"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); Map fateIdsFromResult = getFateIdsFromPrint(result); @@ -378,7 +387,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext */ // Filter by NEW state - p = getCluster().exec(Admin.class, "fate", "--print", "-s", "NEW"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print", + "-s", "NEW"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -386,7 +396,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext fateIdsFromResult); // Filter by FAILED state - p = getCluster().exec(Admin.class, "fate", "--print", "-s", "FAILED"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print", + "-s", "FAILED"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -397,15 +408,16 @@ protected void testFatePrintCommand(FateStore store, ServerContext */ // Filter by one FateId - p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--print"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--print"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); assertEquals(Map.of(fateId1.canonical(), "NEW"), fateIdsFromResult); // Filter by both FateIds - p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), fateId2.canonical(), - "--print"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), fateId2.canonical(), "--print"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -414,7 +426,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext // Filter by non-existent FateId FateId fakeFateId = FateId.from(store.type(), UUID.randomUUID()); - p = getCluster().exec(Admin.class, "fate", fakeFateId.canonical(), "--print"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fakeFateId.canonical(), "--print"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -425,7 +438,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext */ // Test filter by USER FateInstanceType - p = getCluster().exec(Admin.class, "fate", "--print", "-t", "USER"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print", + "-t", "USER"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -437,7 +451,8 @@ protected void testFatePrintCommand(FateStore store, ServerContext } // Test filter by META FateInstanceType - p = getCluster().exec(Admin.class, "fate", "--print", "-t", "META"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print", + "-t", "META"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); fateIdsFromResult = getFateIdsFromPrint(result); @@ -493,7 +508,8 @@ protected void testTransactionNameAndStep(FateStore store, ServerC } List fateIdsStarted = new ArrayList<>(); - ProcessInfo p = getCluster().exec(Admin.class, "fate", "--summary", "-j"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -510,7 +526,7 @@ protected void testTransactionNameAndStep(FateStore store, ServerC } assertEquals(2, fateIdsStarted.size()); - p = getCluster().exec(Admin.class, "fate", "--print"); + p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, "--print"); assertEquals(0, p.getProcess().waitFor()); result = p.readStdOut(); @@ -560,7 +576,8 @@ protected void testFateCancelCommand(FateStore store, ServerContex fateIdsFromSummary); // Cancel the first transaction and ensure that it was cancelled - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--cancel"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--cancel"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -608,7 +625,8 @@ protected void testFateFailCommandTimeout(FateStore store, ServerC // Try to fail fateId1 // This should not work as it is already reserved and being worked on by our running FATE // ('fate'). Admin should try to reserve it for a bit, but should fail and exit - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--fail"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--fail"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -645,7 +663,8 @@ protected void testFateFailCommandSuccess(FateStore store, ServerC // Try to fail fateId1 // This should work since nothing has fateId1 reserved (it is NEW) - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--fail"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--fail"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -694,7 +713,8 @@ protected void testFateDeleteCommandTimeout(FateStore store, Serve // Try to delete fateId1 // This should not work as it is already reserved and being worked on by our running FATE // ('fate'). Admin should try to reserve it for a bit, but should fail and exit - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--delete"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--delete"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -731,7 +751,8 @@ protected void testFateDeleteCommandSuccess(FateStore store, Serve // Try to delete fateId1 // This should work since nothing has fateId1 reserved (it is NEW) - ProcessInfo p = getCluster().exec(Admin.class, "fate", fateId1.canonical(), "--delete"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + fateId1.canonical(), "--delete"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); @@ -873,7 +894,8 @@ private Map getFateIdsFromPrint(String printResult) { * @return a map of each of the FateIds to their status using the --summary command */ private Map getFateIdsFromSummary() throws Exception { - ProcessInfo p = getCluster().exec(Admin.class, "fate", "--summary", "-j"); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + "--summary", "-j"); assertEquals(0, p.getProcess().waitFor()); String result = p.readStdOut(); result = result.lines().filter(line -> !line.matches(".*(INFO|DEBUG|WARN|ERROR).*")) @@ -955,10 +977,10 @@ protected void startManager() throws IOException { protected void cleanupFateOps() throws Exception { List args = new ArrayList<>(); - args.add("fate"); args.addAll(fateOpsToCleanup); args.add("--delete"); - ProcessInfo p = getCluster().exec(Admin.class, args.toArray(new String[0])); + ProcessInfo p = getCluster().exec(org.apache.accumulo.server.util.adminCommand.Fate.class, + args.toArray(new String[0])); assertEquals(0, p.getProcess().waitFor()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java index c1bad36ae49..ff1b0e9c0f3 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java @@ -62,7 +62,7 @@ import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterControl; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopServers; import org.apache.accumulo.test.compaction.ExternalCompactionTestUtils; import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.conf.Configuration; @@ -171,7 +171,7 @@ public void testGracefulShutdown() throws Exception { // Don't call `new Admin().execute(new String[] {"signalShutdown", "-h ", host, "-p ", // Integer.toString(port)})` // because this poisons the SingletonManager and puts it into SERVER mode - Admin.signalGracefulShutdown(ctx, gcAddress); + StopServers.signalGracefulShutdown(ctx, gcAddress); Wait.waitFor(() -> { control.refreshProcesses(ServerType.GARBAGE_COLLECTOR); return control.getProcesses(ServerType.GARBAGE_COLLECTOR).isEmpty(); @@ -183,7 +183,7 @@ public void testGracefulShutdown() throws Exception { assertEquals(2, tservers.size()); final HostAndPort tserverAddress = HostAndPort.fromString(tservers.iterator().next().getServer()); - Admin.signalGracefulShutdown(ctx, tserverAddress); + StopServers.signalGracefulShutdown(ctx, tserverAddress); Wait.waitFor(() -> { control.refreshProcesses(ServerType.TABLET_SERVER); return control.getProcesses(ServerType.TABLET_SERVER).size() == 1; @@ -203,7 +203,7 @@ public void testGracefulShutdown() throws Exception { assertEquals(1, managerLocations.size()); final HostAndPort managerAddress = HostAndPort.fromString(managerLocations.iterator().next().toHostPortString()); - Admin.signalGracefulShutdown(ctx, managerAddress); + StopServers.signalGracefulShutdown(ctx, managerAddress); Wait.waitFor(() -> { control.refreshProcesses(ServerType.MANAGER); return control.getProcesses(ServerType.MANAGER).isEmpty(); @@ -247,7 +247,7 @@ public void testGracefulShutdown() throws Exception { client.tableOperations().compact(tableName, cc); Wait.waitFor(() -> ExternalCompactionTestUtils .getRunningCompactions(ctx, Optional.of(newManagerAddress)).getCompactionsSize() > 0); - Admin.signalGracefulShutdown(ctx, compactorAddress); + StopServers.signalGracefulShutdown(ctx, compactorAddress); Wait.waitFor(() -> { control.refreshProcesses(ServerType.COMPACTOR); return control.getProcesses(ServerType.COMPACTOR).isEmpty(); @@ -279,7 +279,7 @@ public void testGracefulShutdown() throws Exception { assertNotNull(e); count++; if (count == 2) { - Admin.signalGracefulShutdown(ctx, sserver); + StopServers.signalGracefulShutdown(ctx, sserver); } } assertEquals(10, count); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java index 45c62a07bcd..24703efb9e0 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ShutdownIT.java @@ -30,7 +30,8 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; -import org.apache.accumulo.server.util.Admin; +import org.apache.accumulo.server.util.adminCommand.StopAll; +import org.apache.accumulo.server.util.adminCommand.StopServers; import org.apache.accumulo.test.TestIngest; import org.apache.accumulo.test.TestRandomDeletes; import org.apache.accumulo.test.VerifyIngest; @@ -49,7 +50,7 @@ public void shutdownDuringIngest() throws Exception { Process ingest = cluster .exec(TestIngest.class, "-c", cluster.getClientPropsPath(), "--createTable").getProcess(); Thread.sleep(100); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); ingest.destroy(); } @@ -61,7 +62,7 @@ public void shutdownDuringQuery() throws Exception { Process verify = cluster.exec(VerifyIngest.class, "-c", cluster.getClientPropsPath()).getProcess(); Thread.sleep(100); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); verify.destroy(); } @@ -73,7 +74,7 @@ public void shutdownDuringDelete() throws Exception { Process deleter = cluster.exec(TestRandomDeletes.class, "-c", cluster.getClientPropsPath()).getProcess(); Thread.sleep(100); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); deleter.destroy(); } @@ -96,7 +97,7 @@ public void shutdownDuringDeleteTable() throws Exception { }); async.start(); Thread.sleep(100); - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); // give the backfound delete operations a bit to run Thread.sleep(3000); // The delete operations should get stuck or run, but should not throw an exception @@ -108,7 +109,7 @@ public void shutdownDuringDeleteTable() throws Exception { @Test public void stopDuringStart() throws Exception { - assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor()); + assertEquals(0, cluster.exec(StopAll.class).getProcess().waitFor()); } @Test @@ -128,7 +129,7 @@ static void runAdminStopTest(AccumuloClient c, MiniAccumuloClusterImpl cluster) ServerId doomed = tabletServers.iterator().next(); log.info("Stopping " + doomed); assertEquals(0, - cluster.exec(Admin.class, "stop", doomed.toHostPortString()).getProcess().waitFor()); + cluster.exec(StopServers.class, doomed.toHostPortString()).getProcess().waitFor()); Wait.waitFor(() -> c.instanceOperations().getServers(ServerId.Type.TABLET_SERVER).size() == 1); tabletServers = c.instanceOperations().getServers(ServerId.Type.TABLET_SERVER); assertEquals(1, tabletServers.size()); diff --git a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java index 112ba413563..dccd9711876 100644 --- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java +++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java @@ -58,7 +58,6 @@ import org.apache.accumulo.server.conf.util.ZooInfoViewer; import org.apache.accumulo.server.conf.util.ZooPropEditor; import org.apache.accumulo.server.init.Initialize; -import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.CancelCompaction; import org.apache.accumulo.server.util.DumpZookeeper; import org.apache.accumulo.server.util.Info; @@ -68,6 +67,21 @@ import org.apache.accumulo.server.util.UpgradeUtil; import org.apache.accumulo.server.util.ZooKeeperMain; import org.apache.accumulo.server.util.ZooZap; +import org.apache.accumulo.server.util.adminCommand.ChangeSecret; +import org.apache.accumulo.server.util.adminCommand.DeleteZooInstance; +import org.apache.accumulo.server.util.adminCommand.DumpConfig; +import org.apache.accumulo.server.util.adminCommand.Fate; +import org.apache.accumulo.server.util.adminCommand.ListInstances; +import org.apache.accumulo.server.util.adminCommand.ListVolumesUsed; +import org.apache.accumulo.server.util.adminCommand.Locks; +import org.apache.accumulo.server.util.adminCommand.PingServer; +import org.apache.accumulo.server.util.adminCommand.RestoreZookeeper; +import org.apache.accumulo.server.util.adminCommand.ServiceStatus; +import org.apache.accumulo.server.util.adminCommand.StopAll; +import org.apache.accumulo.server.util.adminCommand.StopManager; +import org.apache.accumulo.server.util.adminCommand.StopServers; +import org.apache.accumulo.server.util.adminCommand.SystemCheck; +import org.apache.accumulo.server.util.adminCommand.VerifyTabletAssignments; import org.apache.accumulo.shell.Shell; import org.apache.accumulo.start.Main; import org.apache.accumulo.start.spi.KeywordExecutable; @@ -149,7 +163,8 @@ public int compareTo(CommandInfo o) { public void testExpectedClasses() { assumeTrue(Files.exists(Path.of(System.getProperty("user.dir")).resolve("src"))); SortedSet expectSet = new TreeSet<>(); - expectSet.add(new CommandInfo(UsageGroup.CORE, "admin", Admin.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "change-secret", ChangeSecret.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "check", SystemCheck.class)); expectSet.add( new CommandInfo(UsageGroup.OTHER, "check-compaction-config", CheckCompactionConfig.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "check-accumulo-properties", @@ -157,22 +172,36 @@ public void testExpectedClasses() { expectSet.add(new CommandInfo(UsageGroup.PROCESS, "compactor", CompactorExecutable.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "create-empty", CreateEmpty.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "create-token", CreateToken.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "delete-instance", DeleteZooInstance.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "dump-config", DumpConfig.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "dump-zoo", DumpZookeeper.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "fate", Fate.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "gc", GCExecutable.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "generate-splits", GenerateSplits.class)); expectSet.add(new CommandInfo(UsageGroup.CORE, "help", Help.class)); expectSet.add(new CommandInfo(UsageGroup.CORE, "info", Info.class)); expectSet.add(new CommandInfo(UsageGroup.CORE, "init", Initialize.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "list-instances", ListInstances.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "list-volumes", ListVolumesUsed.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "locks", Locks.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "login-info", LoginProperties.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "manager", ManagerExecutable.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "minicluster", MiniClusterExecutable.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "monitor", MonitorExecutable.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "ping", PingServer.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "restore-zookeeper", RestoreZookeeper.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "rfile-info", PrintInfo.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "service-status", ServiceStatus.class)); expectSet.add(new CommandInfo(UsageGroup.CORE, "shell", Shell.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "split-large", SplitLarge.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "sserver", ScanServerExecutable.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "stop-all", StopAll.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "stop-manager", StopManager.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "stop-servers", StopServers.class)); expectSet.add(new CommandInfo(UsageGroup.PROCESS, "tserver", TServerExecutable.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "upgrade", UpgradeUtil.class)); + expectSet.add(new CommandInfo(UsageGroup.ADMIN, "verify-tablet-assignments", + VerifyTabletAssignments.class)); expectSet.add(new CommandInfo(UsageGroup.CORE, "version", Version.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "wal-info", LogReader.class)); expectSet.add(new CommandInfo(UsageGroup.OTHER, "zoo-info-viewer", ZooInfoViewer.class)); @@ -231,14 +260,16 @@ public void checkHasMain() { "Sanity check for test failed. Somehow the test class has a main method"); HashSet> expectSet = new HashSet<>(); - expectSet.add(Admin.class); expectSet.add(CheckCompactionConfig.class); + expectSet.add(SystemCheck.class); expectSet.add(CreateEmpty.class); expectSet.add(CreateToken.class); + expectSet.add(DumpConfig.class); expectSet.add(DumpZookeeper.class); expectSet.add(GenerateSplits.class); expectSet.add(Info.class); expectSet.add(Initialize.class); + expectSet.add(Fate.class); expectSet.add(LogReader.class); expectSet.add(LoginProperties.class); expectSet.add(MiniAccumuloRunner.class); @@ -246,6 +277,8 @@ public void checkHasMain() { expectSet.add(PrintInfo.class); expectSet.add(Shell.class); expectSet.add(SimpleGarbageCollector.class); + expectSet.add(StopAll.class); + expectSet.add(StopServers.class); expectSet.add(SplitLarge.class); expectSet.add(TabletServer.class); expectSet.add(ZooKeeperMain.class);