From abaa106ab89df3f118fea1ff5428f285fb5fed88 Mon Sep 17 00:00:00 2001 From: Bob Paulin Date: Wed, 1 Jul 2026 17:40:31 -0500 Subject: [PATCH] NIFI-16081: Implement Migrate Connector Properties API * Support for migrateProperties method in Connector * Tests supporting Step and Connector Property updates * System Test covering property migration --- .../MockConnectorPropertyConfiguration.java | 354 ++++++++++++++++++ ...tandardConnectorPropertyConfiguration.java | 169 +++++++++ ...ardConnectorStepPropertyConfiguration.java | 183 +++++++++ .../TestConnectorMigrateProperties.java | 284 ++++++++++++++ ...tandardConnectorPropertyConfiguration.java | 286 ++++++++++++++ .../connector/StandardConnectorNode.java | 64 +++- .../TestStandardConnectorRepository.java | 183 +++++++++ .../system/MigratePropertiesConnector.java | 143 +++++++ .../system/PropertiesParameterProvider.java | 109 ++++++ ...apache.nifi.components.connector.Connector | 16 + ...rg.apache.nifi.parameter.ParameterProvider | 16 + .../system/MigratePropertiesConnector.java | 116 ++++++ ...apache.nifi.components.connector.Connector | 1 + .../ConnectorPropertyMigrationIT.java | 262 +++++++++++++ 14 files changed, 2169 insertions(+), 17 deletions(-) create mode 100644 nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/migration/MockConnectorPropertyConfiguration.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorPropertyConfiguration.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorStepPropertyConfiguration.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestConnectorMigrateProperties.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestStandardConnectorPropertyConfiguration.java create mode 100644 nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java create mode 100644 nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java create mode 100644 nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector create mode 100644 nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorPropertyMigrationIT.java diff --git a/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/migration/MockConnectorPropertyConfiguration.java b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/migration/MockConnectorPropertyConfiguration.java new file mode 100644 index 000000000000..c601ba53aba9 --- /dev/null +++ b/nifi-connector-mock-bundle/nifi-connector-mock/src/main/java/org/apache/nifi/mock/connector/migration/MockConnectorPropertyConfiguration.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.migration; + +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.StringLiteralValue; +import org.apache.nifi.migration.ConnectorPropertyConfiguration; +import org.apache.nifi.migration.ConnectorStepPropertyConfiguration; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Test-only {@link ConnectorPropertyConfiguration} implementation for exercising a Connector's + * {@link org.apache.nifi.components.connector.Connector#migrateProperties(ConnectorPropertyConfiguration) migrateProperties} + * method in unit tests. Tracks per-step property renames, removals, and additions, plus step-level renames, + * removals, and additions. Callers can drive the mock with either a plain string-literal shorthand map or a + * fully typed {@link ConnectorValueReference} map and then inspect the outcome via + * {@link #toMigrationResult()}, {@link #getStepNames()}, and {@link #forStep(String)}. + */ +public class MockConnectorPropertyConfiguration implements ConnectorPropertyConfiguration { + + private final Map> stepProperties = new LinkedHashMap<>(); + private final Set initialStepNames; + + private final Map> propertiesRenamed = new LinkedHashMap<>(); + private final Map> propertiesRemoved = new LinkedHashMap<>(); + private final Map> propertiesUpdated = new LinkedHashMap<>(); + + private final Map renamedSteps = new LinkedHashMap<>(); + private final Set removedSteps = new LinkedHashSet<>(); + private final Set addedSteps = new LinkedHashSet<>(); + + private MockConnectorPropertyConfiguration(final Map> initialProperties) { + if (initialProperties != null) { + for (final Map.Entry> entry : initialProperties.entrySet()) { + final Map copy = new LinkedHashMap<>(); + if (entry.getValue() != null) { + copy.putAll(entry.getValue()); + } + stepProperties.put(entry.getKey(), copy); + } + } + this.initialStepNames = Collections.unmodifiableSet(new LinkedHashSet<>(stepProperties.keySet())); + } + + /** + * Creates a {@link MockConnectorPropertyConfiguration} pre-populated with typed {@link ConnectorValueReference} + * values for each step. + * + * @param initialProperties per-step property maps of {@link ConnectorValueReference} values + * @return a new configuration + */ + public static MockConnectorPropertyConfiguration fromValueReferences(final Map> initialProperties) { + return new MockConnectorPropertyConfiguration(initialProperties); + } + + /** + * Creates a {@link MockConnectorPropertyConfiguration} pre-populated with plain string-literal values for each + * step. Each supplied string is wrapped in a {@link StringLiteralValue}. + * + * @param initialProperties per-step property maps of raw string values + * @return a new configuration + */ + public static MockConnectorPropertyConfiguration fromStringLiterals(final Map> initialProperties) { + final Map> converted = new LinkedHashMap<>(); + if (initialProperties != null) { + for (final Map.Entry> entry : initialProperties.entrySet()) { + final Map stepMap = new LinkedHashMap<>(); + if (entry.getValue() != null) { + for (final Map.Entry propertyEntry : entry.getValue().entrySet()) { + stepMap.put(propertyEntry.getKey(), new StringLiteralValue(propertyEntry.getValue())); + } + } + converted.put(entry.getKey(), stepMap); + } + } + return new MockConnectorPropertyConfiguration(converted); + } + + @Override + public Set getStepNames() { + return Collections.unmodifiableSet(stepProperties.keySet()); + } + + @Override + public boolean hasStep(final String stepName) { + return stepProperties.containsKey(stepName); + } + + @Override + public boolean renameStep(final String oldStepName, final String newStepName) { + if (!stepProperties.containsKey(oldStepName)) { + return false; + } + if (Objects.equals(oldStepName, newStepName)) { + return false; + } + if (stepProperties.containsKey(newStepName)) { + throw new IllegalStateException("Cannot rename step [" + oldStepName + "] to [" + newStepName + + "] because a step with the new name already exists"); + } + + final Map existing = stepProperties.remove(oldStepName); + stepProperties.put(newStepName, existing); + renamedSteps.put(oldStepName, newStepName); + + final Map renames = propertiesRenamed.remove(oldStepName); + if (renames != null) { + propertiesRenamed.put(newStepName, renames); + } + final Set removed = propertiesRemoved.remove(oldStepName); + if (removed != null) { + propertiesRemoved.put(newStepName, removed); + } + final Set updated = propertiesUpdated.remove(oldStepName); + if (updated != null) { + propertiesUpdated.put(newStepName, updated); + } + return true; + } + + @Override + public boolean removeStep(final String stepName) { + if (!stepProperties.containsKey(stepName)) { + return false; + } + stepProperties.remove(stepName); + removedSteps.add(stepName); + propertiesRenamed.remove(stepName); + propertiesRemoved.remove(stepName); + propertiesUpdated.remove(stepName); + return true; + } + + @Override + public ConnectorStepPropertyConfiguration forStep(final String stepName) { + return new MockStepPropertyConfiguration(stepName); + } + + /** + * @return a summary of every mutation observed during migration, suitable for assertions + */ + public MigrationResult toMigrationResult() { + return new MigrationResult( + Collections.unmodifiableMap(new LinkedHashMap<>(renamedSteps)), + Collections.unmodifiableSet(new LinkedHashSet<>(removedSteps)), + Collections.unmodifiableSet(new LinkedHashSet<>(addedSteps)), + deepCopy(propertiesRenamed), + deepCopySets(propertiesRemoved), + deepCopySets(propertiesUpdated) + ); + } + + private static Map> deepCopy(final Map> source) { + final Map> copy = new LinkedHashMap<>(); + for (final Map.Entry> entry : source.entrySet()) { + copy.put(entry.getKey(), Collections.unmodifiableMap(new LinkedHashMap<>(entry.getValue()))); + } + return Collections.unmodifiableMap(copy); + } + + private static Map> deepCopySets(final Map> source) { + final Map> copy = new LinkedHashMap<>(); + for (final Map.Entry> entry : source.entrySet()) { + copy.put(entry.getKey(), Collections.unmodifiableSet(new LinkedHashSet<>(entry.getValue()))); + } + return Collections.unmodifiableMap(copy); + } + + private Map getOrCreateStepMap(final String stepName) { + return stepProperties.computeIfAbsent(stepName, key -> { + if (!initialStepNames.contains(key)) { + addedSteps.add(key); + } + return new LinkedHashMap<>(); + }); + } + + private void trackRenamed(final String stepName, final String oldName, final String newName) { + propertiesRenamed.computeIfAbsent(stepName, key -> new LinkedHashMap<>()).put(oldName, newName); + } + + private void trackRemoved(final String stepName, final String propertyName) { + propertiesRemoved.computeIfAbsent(stepName, key -> new LinkedHashSet<>()).add(propertyName); + } + + private void trackUpdated(final String stepName, final String propertyName) { + propertiesUpdated.computeIfAbsent(stepName, key -> new LinkedHashSet<>()).add(propertyName); + } + + /** + * A summary of every step and property mutation observed during a call to + * {@link org.apache.nifi.components.connector.Connector#migrateProperties(ConnectorPropertyConfiguration)}. + * + * @param renamedSteps old step name to new step name for every renamed step + * @param removedSteps step names removed during migration + * @param addedSteps step names introduced during migration + * @param propertiesRenamed per-step map from old property name to new property name + * @param propertiesRemoved per-step set of property names removed + * @param propertiesUpdated per-step set of property names added or overwritten (including via setValueReference) + */ + public record MigrationResult( + Map renamedSteps, + Set removedSteps, + Set addedSteps, + Map> propertiesRenamed, + Map> propertiesRemoved, + Map> propertiesUpdated + ) { + } + + private final class MockStepPropertyConfiguration implements ConnectorStepPropertyConfiguration { + private final String stepName; + + private MockStepPropertyConfiguration(final String stepName) { + this.stepName = stepName; + } + + @Override + public String getStepName() { + return stepName; + } + + @Override + public boolean renameProperty(final String propertyName, final String newName) { + trackRenamed(stepName, propertyName, newName); + final Map properties = stepProperties.get(stepName); + if (properties == null || !properties.containsKey(propertyName)) { + return false; + } + if (Objects.equals(propertyName, newName)) { + return false; + } + final ConnectorValueReference existing = properties.remove(propertyName); + properties.put(newName, existing); + return true; + } + + @Override + public boolean removeProperty(final String propertyName) { + trackRemoved(stepName, propertyName); + final Map properties = stepProperties.get(stepName); + if (properties == null || !properties.containsKey(propertyName)) { + return false; + } + properties.remove(propertyName); + return true; + } + + @Override + public boolean hasProperty(final String propertyName) { + final Map properties = stepProperties.get(stepName); + return properties != null && properties.containsKey(propertyName); + } + + @Override + public boolean isPropertySet(final String propertyName) { + final Map properties = stepProperties.get(stepName); + if (properties == null) { + return false; + } + final ConnectorValueReference reference = properties.get(propertyName); + if (reference == null) { + return false; + } + if (reference instanceof StringLiteralValue literal) { + return literal.getValue() != null; + } + return true; + } + + @Override + public void setProperty(final String propertyName, final String propertyValue) { + setValueReference(propertyName, new StringLiteralValue(propertyValue)); + } + + @Override + public void setValueReference(final String propertyName, final ConnectorValueReference valueReference) { + if (valueReference == null) { + removeProperty(propertyName); + return; + } + trackUpdated(stepName, propertyName); + getOrCreateStepMap(stepName).put(propertyName, valueReference); + } + + @Override + public Optional getPropertyValue(final String propertyName) { + final Map properties = stepProperties.get(stepName); + if (properties == null) { + return Optional.empty(); + } + final ConnectorValueReference reference = properties.get(propertyName); + if (reference instanceof StringLiteralValue literal) { + return Optional.ofNullable(literal.getValue()); + } + return Optional.empty(); + } + + @Override + public Optional getValueReference(final String propertyName) { + final Map properties = stepProperties.get(stepName); + if (properties == null) { + return Optional.empty(); + } + return Optional.ofNullable(properties.get(propertyName)); + } + + @Override + public Map getProperties() { + final Map properties = stepProperties.get(stepName); + if (properties == null || properties.isEmpty()) { + return Collections.emptyMap(); + } + final Map literals = new LinkedHashMap<>(); + for (final Map.Entry entry : properties.entrySet()) { + if (entry.getValue() instanceof StringLiteralValue literal) { + literals.put(entry.getKey(), literal.getValue()); + } + } + return Collections.unmodifiableMap(literals); + } + + @Override + public Map getValueReferences() { + final Map properties = stepProperties.get(stepName); + if (properties == null) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorPropertyConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorPropertyConfiguration.java new file mode 100644 index 000000000000..0e53ffb89d9a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorPropertyConfiguration.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.migration; + +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Framework implementation of {@link ConnectorPropertyConfiguration}. Holds the per-step property map for a single + * {@link org.apache.nifi.components.connector.Connector Connector} configuration snapshot (active or working) and + * tracks whether any modification has been made so callers can conditionally persist the migrated state. + * + *

+ * Instances are not thread-safe; a Connector's {@code migrateProperties} invocation is expected to be + * single-threaded. + *

+ */ +public class StandardConnectorPropertyConfiguration implements ConnectorPropertyConfiguration { + + private static final Logger logger = LoggerFactory.getLogger(StandardConnectorPropertyConfiguration.class); + + private final Map> stepProperties; + private final Set modifiedStepNames = new LinkedHashSet<>(); + private final String componentDescription; + private boolean modified = false; + + public StandardConnectorPropertyConfiguration(final Map> initialProperties, final String componentDescription) { + this.componentDescription = componentDescription; + this.stepProperties = new LinkedHashMap<>(); + if (initialProperties != null) { + for (final Map.Entry> entry : initialProperties.entrySet()) { + final Map copy = new LinkedHashMap<>(); + if (entry.getValue() != null) { + copy.putAll(entry.getValue()); + } + stepProperties.put(entry.getKey(), copy); + } + } + } + + @Override + public Set getStepNames() { + return Collections.unmodifiableSet(stepProperties.keySet()); + } + + @Override + public boolean hasStep(final String stepName) { + return stepProperties.containsKey(stepName); + } + + @Override + public boolean renameStep(final String oldStepName, final String newStepName) { + if (!stepProperties.containsKey(oldStepName)) { + logger.debug("Will not rename step [{}] for [{}] because the step is not known", oldStepName, componentDescription); + return false; + } + if (Objects.equals(oldStepName, newStepName)) { + logger.debug("Will not rename step [{}] for [{}] because the new name matches the current name", oldStepName, componentDescription); + return false; + } + if (stepProperties.containsKey(newStepName)) { + throw new IllegalStateException("Cannot rename configuration step [" + oldStepName + "] to [" + newStepName + + "] for [" + componentDescription + "] because a step with the new name already exists"); + } + + final Map existing = stepProperties.remove(oldStepName); + stepProperties.put(newStepName, existing); + markModified(oldStepName); + markModified(newStepName); + logger.info("Renamed configuration step [{}] to [{}] for [{}]", oldStepName, newStepName, componentDescription); + return true; + } + + @Override + public boolean removeStep(final String stepName) { + if (!stepProperties.containsKey(stepName)) { + logger.debug("Will not remove step [{}] for [{}] because the step is not known", stepName, componentDescription); + return false; + } + + stepProperties.remove(stepName); + markModified(stepName); + logger.info("Removed configuration step [{}] for [{}]", stepName, componentDescription); + return true; + } + + @Override + public ConnectorStepPropertyConfiguration forStep(final String stepName) { + return new StandardConnectorStepPropertyConfiguration(stepName, this, componentDescription); + } + + /** + * @return true if any step or property modification has been made through this configuration + */ + public boolean isModified() { + return modified; + } + + /** + * @return the names of steps that were mutated by this configuration (including steps that were removed) + */ + public Set getModifiedStepNames() { + return Collections.unmodifiableSet(modifiedStepNames); + } + + /** + * @return the authoritative post-migration per-step property map for this configuration snapshot + */ + public Map> getMutatedProperties() { + final Map> snapshot = new LinkedHashMap<>(); + for (final Map.Entry> entry : stepProperties.entrySet()) { + snapshot.put(entry.getKey(), new LinkedHashMap<>(entry.getValue())); + } + return snapshot; + } + + /** + * Returns the mutable per-step property map for the given step, or null if the step is not currently + * registered. Used by {@link StandardConnectorStepPropertyConfiguration} for read-only lookups. + */ + Map getStepMap(final String stepName) { + return stepProperties.get(stepName); + } + + /** + * Returns the mutable per-step property map for the given step, allocating and registering a new empty map when + * the step is not currently registered. Used by {@link StandardConnectorStepPropertyConfiguration} for the first + * write into a lazily created step. + */ + Map getOrCreateStepMap(final String stepName) { + return stepProperties.computeIfAbsent(stepName, key -> { + logger.info("Registered new configuration step [{}] for [{}]", stepName, componentDescription); + return new LinkedHashMap<>(); + }); + } + + /** + * Flags the configuration and the given step as modified. Invoked by + * {@link StandardConnectorStepPropertyConfiguration} on every mutating operation and by the top-level step + * operations on this class. + */ + void markModified(final String stepName) { + modified = true; + modifiedStepNames.add(stepName); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorStepPropertyConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorStepPropertyConfiguration.java new file mode 100644 index 000000000000..ab875669c0e4 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/migration/StandardConnectorStepPropertyConfiguration.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.migration; + +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.StringLiteralValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Framework implementation of {@link ConnectorStepPropertyConfiguration}. Instances are created and managed by + * {@link StandardConnectorPropertyConfiguration}; they mutate the parent's per-step map on write and lazily register + * a new step in the parent's step map the first time a property is written into it. + */ +public class StandardConnectorStepPropertyConfiguration implements ConnectorStepPropertyConfiguration { + + private static final Logger logger = LoggerFactory.getLogger(StandardConnectorStepPropertyConfiguration.class); + + private final String stepName; + private final StandardConnectorPropertyConfiguration parent; + private final String componentDescription; + + StandardConnectorStepPropertyConfiguration(final String stepName, final StandardConnectorPropertyConfiguration parent, final String componentDescription) { + this.stepName = stepName; + this.parent = parent; + this.componentDescription = componentDescription; + } + + @Override + public String getStepName() { + return stepName; + } + + @Override + public boolean renameProperty(final String propertyName, final String newName) { + final Map properties = parent.getStepMap(stepName); + if (properties == null || !properties.containsKey(propertyName)) { + logger.debug("Will not rename property [{}] in step [{}] for [{}] because the property is not known", propertyName, stepName, componentDescription); + return false; + } + + if (Objects.equals(propertyName, newName)) { + logger.debug("Will not rename property [{}] in step [{}] for [{}] because the new name matches the current name", propertyName, stepName, componentDescription); + return false; + } + + final ConnectorValueReference existing = properties.remove(propertyName); + properties.put(newName, existing); + parent.markModified(stepName); + logger.info("Renamed property [{}] to [{}] in step [{}] for [{}]", propertyName, newName, stepName, componentDescription); + return true; + } + + @Override + public boolean removeProperty(final String propertyName) { + final Map properties = parent.getStepMap(stepName); + if (properties == null || !properties.containsKey(propertyName)) { + logger.debug("Will not remove property [{}] from step [{}] for [{}] because the property is not known", propertyName, stepName, componentDescription); + return false; + } + + properties.remove(propertyName); + parent.markModified(stepName); + logger.info("Removed property [{}] from step [{}] for [{}]", propertyName, stepName, componentDescription); + return true; + } + + @Override + public boolean hasProperty(final String propertyName) { + final Map properties = parent.getStepMap(stepName); + return properties != null && properties.containsKey(propertyName); + } + + @Override + public boolean isPropertySet(final String propertyName) { + final Map properties = parent.getStepMap(stepName); + if (properties == null) { + return false; + } + final ConnectorValueReference reference = properties.get(propertyName); + if (reference == null) { + return false; + } + if (reference instanceof StringLiteralValue literal) { + return literal.getValue() != null; + } + return true; + } + + @Override + public void setProperty(final String propertyName, final String propertyValue) { + setValueReference(propertyName, new StringLiteralValue(propertyValue)); + } + + @Override + public void setValueReference(final String propertyName, final ConnectorValueReference valueReference) { + if (valueReference == null) { + removeProperty(propertyName); + return; + } + + final Map properties = parent.getOrCreateStepMap(stepName); + final ConnectorValueReference previous = properties.put(propertyName, valueReference); + if (Objects.equals(previous, valueReference)) { + logger.debug("Will not update property [{}] in step [{}] for [{}] because the proposed value matches the current value", propertyName, stepName, componentDescription); + return; + } + + parent.markModified(stepName); + if (previous == null) { + logger.info("Updated property [{}] in step [{}] for [{}], which was previously unset", propertyName, stepName, componentDescription); + } else { + logger.info("Updated property [{}] in step [{}] for [{}], overwriting previous value", propertyName, stepName, componentDescription); + } + } + + @Override + public Optional getPropertyValue(final String propertyName) { + final Map properties = parent.getStepMap(stepName); + if (properties == null) { + return Optional.empty(); + } + final ConnectorValueReference reference = properties.get(propertyName); + if (reference instanceof StringLiteralValue literal) { + return Optional.ofNullable(literal.getValue()); + } + return Optional.empty(); + } + + @Override + public Optional getValueReference(final String propertyName) { + final Map properties = parent.getStepMap(stepName); + if (properties == null) { + return Optional.empty(); + } + return Optional.ofNullable(properties.get(propertyName)); + } + + @Override + public Map getProperties() { + final Map properties = parent.getStepMap(stepName); + if (properties == null || properties.isEmpty()) { + return Collections.emptyMap(); + } + final Map literals = new LinkedHashMap<>(); + for (final Map.Entry entry : properties.entrySet()) { + if (entry.getValue() instanceof StringLiteralValue literal) { + literals.put(entry.getKey(), literal.getValue()); + } + } + return Collections.unmodifiableMap(literals); + } + + @Override + public Map getValueReferences() { + final Map properties = parent.getStepMap(stepName); + if (properties == null) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(properties); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestConnectorMigrateProperties.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestConnectorMigrateProperties.java new file mode 100644 index 000000000000..88463a83cdae --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestConnectorMigrateProperties.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.migration; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StringLiteralValue; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises {@link org.apache.nifi.components.connector.Connector#migrateProperties(ConnectorPropertyConfiguration)} + * from real {@link AbstractConnector} subclasses. Each scenario is expressed as a tiny standalone connector whose only + * behaviour is a single {@code migrateProperties} override, driven against a live + * {@link StandardConnectorPropertyConfiguration}, so the tests double as canonical usage examples for connector + * authors. + */ +public class TestConnectorMigrateProperties { + + private static final String STEP_A = "Step A"; + private static final String LEGACY_STEP = "Legacy Step"; + private static final String OBSOLETE_STEP = "Obsolete Step"; + private static final String COMPONENT_DESCRIPTION = "Test Connector"; + + @Test + public void testAddProperty() { + final Map> initial = new LinkedHashMap<>(); + final Map stepMap = new LinkedHashMap<>(); + stepMap.put("existing", new StringLiteralValue("keep")); + initial.put(STEP_A, stepMap); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new AddPropertyConnector().migrateProperties(config); + + assertTrue(config.isModified()); + final Map migrated = config.forStep(STEP_A).getValueReferences(); + assertTrue(migrated.containsKey("added")); + assertInstanceOf(StringLiteralValue.class, migrated.get("added")); + assertEquals("value", ((StringLiteralValue) migrated.get("added")).getValue()); + assertTrue(migrated.containsKey("existing")); + } + + @Test + public void testRemoveProperty() { + final Map> initial = new LinkedHashMap<>(); + final Map stepMap = new LinkedHashMap<>(); + stepMap.put("legacy", new StringLiteralValue("unused")); + stepMap.put("keep", new StringLiteralValue("kept")); + initial.put(STEP_A, stepMap); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new RemovePropertyConnector().migrateProperties(config); + + assertTrue(config.isModified()); + assertFalse(config.forStep(STEP_A).hasProperty("legacy")); + assertTrue(config.forStep(STEP_A).hasProperty("keep")); + } + + @Test + public void testRenamePropertyPreservesStringLiteral() { + final Map> initial = new LinkedHashMap<>(); + final Map stepMap = new LinkedHashMap<>(); + stepMap.put("old", new StringLiteralValue("value")); + initial.put(STEP_A, stepMap); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new RenamePropertyStringConnector().migrateProperties(config); + + assertTrue(config.isModified()); + final Map migrated = config.forStep(STEP_A).getValueReferences(); + assertFalse(migrated.containsKey("old")); + assertInstanceOf(StringLiteralValue.class, migrated.get("new")); + assertEquals("value", ((StringLiteralValue) migrated.get("new")).getValue()); + } + + @Test + public void testRenamePropertyPreservesSecretReference() { + final SecretReference secret = new SecretReference("vault", "Vault", "credentials/path", "vault:credentials/path"); + final Map> initial = new LinkedHashMap<>(); + final Map stepMap = new LinkedHashMap<>(); + stepMap.put("old-secret", secret); + initial.put(STEP_A, stepMap); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new RenamePropertySecretConnector().migrateProperties(config); + + assertTrue(config.isModified()); + final ConnectorValueReference migrated = config.forStep(STEP_A).getValueReference("credentials").orElseThrow(); + assertInstanceOf(SecretReference.class, migrated); + assertSame(secret, migrated); + } + + @Test + public void testRenameStepCarriesAllPropertiesIntact() { + final SecretReference secret = new SecretReference("vault", "Vault", "credentials/path", "vault:credentials/path"); + final Map legacyProperties = new LinkedHashMap<>(); + legacyProperties.put("string-prop", new StringLiteralValue("string-value")); + legacyProperties.put("secret-prop", secret); + + final Map> initial = new LinkedHashMap<>(); + initial.put(LEGACY_STEP, legacyProperties); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new RenameStepConnector().migrateProperties(config); + + assertTrue(config.isModified()); + assertFalse(config.hasStep(LEGACY_STEP)); + assertTrue(config.hasStep(STEP_A)); + final Map migrated = config.forStep(STEP_A).getValueReferences(); + assertEquals(legacyProperties, migrated); + assertSame(secret, migrated.get("secret-prop")); + } + + @Test + public void testRemoveStepDropsAllProperties() { + final Map> initial = new LinkedHashMap<>(); + final Map obsolete = new LinkedHashMap<>(); + obsolete.put("prop", new StringLiteralValue("value")); + initial.put(OBSOLETE_STEP, obsolete); + final Map survivor = new LinkedHashMap<>(); + survivor.put("keep", new StringLiteralValue("kept")); + initial.put(STEP_A, survivor); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new RemoveStepConnector().migrateProperties(config); + + assertTrue(config.isModified()); + assertFalse(config.hasStep(OBSOLETE_STEP)); + assertEquals(Set.of(STEP_A), config.getStepNames()); + } + + @Test + public void testCompositeMigrationAcrossAllScenarios() { + final SecretReference secret = new SecretReference("vault", "Vault", "credentials/path", "vault:credentials/path"); + final Map legacyProperties = new LinkedHashMap<>(); + legacyProperties.put("old", new StringLiteralValue("string-value")); + legacyProperties.put("old-secret", secret); + legacyProperties.put("obsolete", new StringLiteralValue("drop-me")); + + final Map obsoleteStep = new LinkedHashMap<>(); + obsoleteStep.put("prop", new StringLiteralValue("gone")); + + final Map> initial = new LinkedHashMap<>(); + initial.put(LEGACY_STEP, legacyProperties); + initial.put(OBSOLETE_STEP, obsoleteStep); + + final StandardConnectorPropertyConfiguration config = new StandardConnectorPropertyConfiguration(initial, COMPONENT_DESCRIPTION); + new FullHistoryConnector().migrateProperties(config); + + assertTrue(config.isModified()); + assertFalse(config.hasStep(LEGACY_STEP)); + assertFalse(config.hasStep(OBSOLETE_STEP)); + assertEquals(Set.of(STEP_A), config.getStepNames()); + + final Map migrated = config.forStep(STEP_A).getValueReferences(); + assertEquals(Set.of("new", "credentials", "added"), migrated.keySet()); + assertInstanceOf(StringLiteralValue.class, migrated.get("new")); + assertEquals("string-value", ((StringLiteralValue) migrated.get("new")).getValue()); + assertSame(secret, migrated.get("credentials")); + assertInstanceOf(StringLiteralValue.class, migrated.get("added")); + assertEquals("value", ((StringLiteralValue) migrated.get("added")).getValue()); + } + + /** + * Base connector for migration scenarios. Provides no-op implementations of every abstract member of + * {@link org.apache.nifi.components.connector.Connector} and {@link AbstractConnector} other than + * {@code migrateProperties}, so scenario subclasses only need to express the migration itself. + */ + private abstract static class MigrationScenarioConnector extends AbstractConnector { + @Override + public VersionedExternalFlow getInitialFlow() { + return null; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return null; + } + + @Override + public List getConfigurationSteps() { + return List.of(); + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) throws FlowUpdateException { + } + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) throws FlowUpdateException { + } + } + + private static final class AddPropertyConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.forStep(STEP_A).setProperty("added", "value"); + } + } + + private static final class RemovePropertyConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.forStep(STEP_A).removeProperty("legacy"); + } + } + + private static final class RenamePropertyStringConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.forStep(STEP_A).renameProperty("old", "new"); + } + } + + private static final class RenamePropertySecretConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.forStep(STEP_A).renameProperty("old-secret", "credentials"); + } + } + + private static final class RenameStepConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.renameStep(LEGACY_STEP, STEP_A); + } + } + + private static final class RemoveStepConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.removeStep(OBSOLETE_STEP); + } + } + + private static final class FullHistoryConnector extends MigrationScenarioConnector { + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.renameStep(LEGACY_STEP, STEP_A); + final ConnectorStepPropertyConfiguration stepA = config.forStep(STEP_A); + stepA.renameProperty("old", "new"); + stepA.renameProperty("old-secret", "credentials"); + stepA.removeProperty("obsolete"); + stepA.setProperty("added", "value"); + config.removeStep(OBSOLETE_STEP); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestStandardConnectorPropertyConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestStandardConnectorPropertyConfiguration.java new file mode 100644 index 000000000000..85b4b95b8faf --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/migration/TestStandardConnectorPropertyConfiguration.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.migration; + +import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StringLiteralValue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestStandardConnectorPropertyConfiguration { + + private static final String STEP_ONE = "Step One"; + private static final String STEP_TWO = "Step Two"; + + private StandardConnectorPropertyConfiguration config; + + @BeforeEach + public void setup() { + final Map> initial = new LinkedHashMap<>(); + final Map stepOne = new LinkedHashMap<>(); + stepOne.put("literal", new StringLiteralValue("value")); + stepOne.put("secret", new SecretReference("provider-id", "Provider", "top-secret", "provider:top-secret")); + stepOne.put("null-literal", StringLiteralValue.EMPTY); + initial.put(STEP_ONE, stepOne); + + final Map stepTwo = new LinkedHashMap<>(); + stepTwo.put("asset", new AssetReference(Set.of("asset-id-1"))); + initial.put(STEP_TWO, stepTwo); + + config = new StandardConnectorPropertyConfiguration(initial, "Test Connector"); + } + + @Test + public void testInitialState() { + assertFalse(config.isModified()); + assertEquals(Set.of(STEP_ONE, STEP_TWO), config.getStepNames()); + assertTrue(config.hasStep(STEP_ONE)); + assertFalse(config.hasStep("Missing Step")); + assertTrue(config.getModifiedStepNames().isEmpty()); + } + + @Test + public void testForStepReadOnlyLookups() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + assertEquals(STEP_ONE, stepOne.getStepName()); + assertTrue(stepOne.hasProperty("literal")); + assertTrue(stepOne.isPropertySet("literal")); + assertTrue(stepOne.hasProperty("null-literal")); + assertFalse(stepOne.isPropertySet("null-literal")); + assertEquals(Optional.of("value"), stepOne.getPropertyValue("literal")); + assertEquals(Optional.empty(), stepOne.getPropertyValue("null-literal")); + assertEquals(Optional.empty(), stepOne.getPropertyValue("secret")); + assertInstanceOf(SecretReference.class, stepOne.getValueReference("secret").orElseThrow()); + + final Map literalProperties = stepOne.getProperties(); + assertEquals(2, literalProperties.size()); + assertEquals("value", literalProperties.get("literal")); + assertTrue(literalProperties.containsKey("null-literal")); + assertNull(literalProperties.get("null-literal")); + assertEquals(3, stepOne.getValueReferences().size()); + } + + @Test + public void testAddProperty() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + stepOne.setProperty("added", "value"); + + assertTrue(config.isModified()); + assertEquals(Set.of(STEP_ONE), config.getModifiedStepNames()); + assertTrue(stepOne.hasProperty("added")); + assertEquals(Optional.of("value"), stepOne.getPropertyValue("added")); + } + + @Test + public void testAddPropertyToNewStepLazyRegisters() { + assertFalse(config.hasStep("New Step")); + final ConnectorStepPropertyConfiguration newStep = config.forStep("New Step"); + + assertFalse(config.hasStep("New Step")); + assertFalse(config.isModified()); + + newStep.setProperty("prop", "value"); + + assertTrue(config.hasStep("New Step")); + assertTrue(config.isModified()); + assertEquals(Optional.of("value"), config.forStep("New Step").getPropertyValue("prop")); + } + + @Test + public void testForStepWithoutWriteDoesNotRegister() { + config.forStep("Unseen"); + assertFalse(config.hasStep("Unseen")); + assertFalse(config.isModified()); + } + + @Test + public void testRemoveProperty() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + assertFalse(stepOne.removeProperty("missing")); + assertFalse(config.isModified()); + + assertTrue(stepOne.removeProperty("literal")); + assertTrue(config.isModified()); + assertFalse(stepOne.hasProperty("literal")); + } + + @Test + public void testRenamePropertyPreservesStringLiteral() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + assertTrue(stepOne.renameProperty("literal", "renamed")); + + assertTrue(config.isModified()); + assertFalse(stepOne.hasProperty("literal")); + assertEquals(Optional.of("value"), stepOne.getPropertyValue("renamed")); + assertInstanceOf(StringLiteralValue.class, stepOne.getValueReference("renamed").orElseThrow()); + } + + @Test + public void testRenamePropertyPreservesSecretReference() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + final ConnectorValueReference originalSecret = stepOne.getValueReference("secret").orElseThrow(); + + assertTrue(stepOne.renameProperty("secret", "renamed-secret")); + + final ConnectorValueReference renamedSecret = stepOne.getValueReference("renamed-secret").orElseThrow(); + assertInstanceOf(SecretReference.class, renamedSecret); + assertSame(originalSecret, renamedSecret); + } + + @Test + public void testRenamePropertyPreservesAssetReference() { + final ConnectorStepPropertyConfiguration stepTwo = config.forStep(STEP_TWO); + final ConnectorValueReference originalAsset = stepTwo.getValueReference("asset").orElseThrow(); + + assertTrue(stepTwo.renameProperty("asset", "renamed-asset")); + + final ConnectorValueReference renamedAsset = stepTwo.getValueReference("renamed-asset").orElseThrow(); + assertInstanceOf(AssetReference.class, renamedAsset); + assertSame(originalAsset, renamedAsset); + } + + @Test + public void testRenamePropertyToSameNameIsNoOp() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + assertFalse(stepOne.renameProperty("literal", "literal")); + assertFalse(config.isModified()); + } + + @Test + public void testRenameUnknownPropertyReturnsFalse() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + assertFalse(stepOne.renameProperty("missing", "renamed")); + assertFalse(config.isModified()); + } + + @Test + public void testSetValueReferenceForEachType() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + final SecretReference newSecret = new SecretReference("p", "P", "sec", "P:sec"); + stepOne.setValueReference("added-secret", newSecret); + assertSame(newSecret, stepOne.getValueReference("added-secret").orElseThrow()); + + final AssetReference newAsset = new AssetReference(Set.of("id-2")); + stepOne.setValueReference("added-asset", newAsset); + assertSame(newAsset, stepOne.getValueReference("added-asset").orElseThrow()); + + final StringLiteralValue newLiteral = new StringLiteralValue("literal-value"); + stepOne.setValueReference("added-literal", newLiteral); + assertEquals(Optional.of("literal-value"), stepOne.getPropertyValue("added-literal")); + } + + @Test + public void testSetValueReferenceNullRemovesProperty() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + stepOne.setValueReference("literal", null); + assertFalse(stepOne.hasProperty("literal")); + assertTrue(config.isModified()); + } + + @Test + public void testRenameStepMovesAllProperties() { + final Map originalSnapshot = new LinkedHashMap<>(config.forStep(STEP_ONE).getValueReferences()); + + assertTrue(config.renameStep(STEP_ONE, "Renamed")); + + assertTrue(config.isModified()); + assertFalse(config.hasStep(STEP_ONE)); + assertTrue(config.hasStep("Renamed")); + final ConnectorStepPropertyConfiguration renamed = config.forStep("Renamed"); + assertEquals(originalSnapshot, renamed.getValueReferences()); + for (final Map.Entry entry : originalSnapshot.entrySet()) { + assertSame(entry.getValue(), renamed.getValueReference(entry.getKey()).orElse(null)); + } + } + + @Test + public void testRenameStepToSameNameIsNoOp() { + assertFalse(config.renameStep(STEP_ONE, STEP_ONE)); + assertFalse(config.isModified()); + } + + @Test + public void testRenameStepToExistingNameThrows() { + assertThrows(IllegalStateException.class, () -> config.renameStep(STEP_ONE, STEP_TWO)); + } + + @Test + public void testRenameUnknownStepReturnsFalse() { + assertFalse(config.renameStep("Missing", "Whatever")); + assertFalse(config.isModified()); + } + + @Test + public void testRemoveStep() { + assertFalse(config.removeStep("Missing")); + assertFalse(config.isModified()); + + assertTrue(config.removeStep(STEP_ONE)); + assertTrue(config.isModified()); + assertFalse(config.hasStep(STEP_ONE)); + assertFalse(config.getStepNames().contains(STEP_ONE)); + } + + @Test + public void testGetPropertiesFiltersNonLiteralAndPreservesNulls() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + final Map literals = stepOne.getProperties(); + assertEquals(2, literals.size()); + assertEquals("value", literals.get("literal")); + assertTrue(literals.containsKey("null-literal")); + assertNull(literals.get("null-literal")); + assertFalse(literals.containsKey("secret")); + } + + @Test + public void testGetMutatedPropertiesReflectsFinalState() { + final ConnectorStepPropertyConfiguration stepOne = config.forStep(STEP_ONE); + stepOne.renameProperty("literal", "renamed"); + stepOne.removeProperty("secret"); + stepOne.setProperty("added", "new-value"); + config.removeStep(STEP_TWO); + config.forStep("Fresh Step").setProperty("prop", "value"); + + final Map> mutated = config.getMutatedProperties(); + assertEquals(Set.of(STEP_ONE, "Fresh Step"), mutated.keySet()); + + final Map stepOneMap = mutated.get(STEP_ONE); + assertTrue(stepOneMap.containsKey("renamed")); + assertFalse(stepOneMap.containsKey("literal")); + assertFalse(stepOneMap.containsKey("secret")); + assertTrue(stepOneMap.containsKey("added")); + assertTrue(stepOneMap.containsKey("null-literal")); + + assertEquals(1, mutated.get("Fresh Step").size()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java index c9f8c9d3769f..784ad47e3a6e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java @@ -54,6 +54,7 @@ import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.GroupedComponent; +import org.apache.nifi.migration.StandardConnectorPropertyConfiguration; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.util.StringUtils; @@ -70,6 +71,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -310,41 +312,69 @@ public void inheritConfiguration(final List activeCo final Bundle flowContextBundle) throws FlowUpdateException { logger.debug("Inheriting configuration for {}", this); - final MutableConnectorConfigurationContext configurationContext = createConfigurationContext(activeConfig); + + // Give the Connector a chance to evolve its persisted property/step names before we build the runtime + // configuration contexts. Active and working configs are migrated independently because they can diverge. + final Map> migratedActiveProperties = migrateProperties(activeConfig); + final Map> migratedWorkingProperties = migrateProperties(workingConfig); + + final MutableConnectorConfigurationContext activeSeedContext = createConfigurationContext(migratedActiveProperties); final FrameworkFlowContext inheritContext = flowContextFactory.createWorkingFlowContext(identifier, - connectorDetails.getComponentLog(), configurationContext, flowContextBundle); + connectorDetails.getComponentLog(), activeSeedContext, flowContextBundle); - // Apply the update for the active config + // Apply the active configuration. This restores activeFlowContext to migratedActiveProperties and internally + // rebuilds workingFlowContext aliased to activeFlowContext's configuration; we discard that alias below and + // construct an independent working context so active and working do not share configuration state when the + // two lists actually diverge. applyUpdate(inheritContext); - // Configure the working config but do not apply - for (final VersionedConfigurationStep step : workingConfig) { - final StepConfiguration stepConfig = createStepConfiguration(step); - setConfiguration(step.getName(), stepConfig, true); + // Tear down the working context that applyUpdate created aliased to active, and rebuild it around an + // independent configuration seeded from migratedWorkingProperties. Then fire onConfigurationStepConfigured + // for every step so renamed steps trigger the flow-builder callback under their new name and any + // value-derived flow state (resolved asset paths, secret values, etc.) is populated against the fresh + // working context. + destroyWorkingContext(); + final MutableConnectorConfigurationContext workingConfigContext = createConfigurationContext(migratedWorkingProperties); + workingFlowContext = flowContextFactory.createWorkingFlowContext(identifier, + connectorDetails.getComponentLog(), workingConfigContext, flowContextBundle); + getComponentLog().info("Working Flow Context has been rebuilt with independent configuration"); + for (final String stepName : migratedWorkingProperties.keySet()) { + notifyStepConfigured(stepName); } logger.debug("Successfully inherited configuration for {}", this); } - private StepConfiguration createStepConfiguration(final VersionedConfigurationStep step) { - final Map convertedProperties = new HashMap<>(); + private Map> migrateProperties(final List flowConfiguration) { + final Map> initial = new LinkedHashMap<>(); + for (final VersionedConfigurationStep versionedConfigStep : flowConfiguration) { + initial.put(versionedConfigStep.getName(), toValueReferenceMap(versionedConfigStep)); + } + + final StandardConnectorPropertyConfiguration propertyConfiguration = new StandardConnectorPropertyConfiguration(initial, this.toString()); + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, getConnector().getClass(), getIdentifier())) { + getConnector().migrateProperties(propertyConfiguration); + } + return propertyConfiguration.getMutatedProperties(); + } + + private Map toValueReferenceMap(final VersionedConfigurationStep step) { + final Map convertedProperties = new LinkedHashMap<>(); if (step.getProperties() != null) { for (final Map.Entry entry : step.getProperties().entrySet()) { - final ConnectorValueReference valueReference = createValueReference(entry.getValue()); - convertedProperties.put(entry.getKey(), valueReference); + convertedProperties.put(entry.getKey(), createValueReference(entry.getValue())); } } - - return new StepConfiguration(convertedProperties); + return convertedProperties; } - private MutableConnectorConfigurationContext createConfigurationContext(final List flowConfiguration) { + private MutableConnectorConfigurationContext createConfigurationContext(final Map> migratedConfiguration) { final StandardConnectorConfigurationContext configurationContext = new StandardConnectorConfigurationContext( initializationContext.getAssetManager(), initializationContext.getSecretsManager()); - for (final VersionedConfigurationStep versionedConfigStep : flowConfiguration) { - final StepConfiguration stepConfig = createStepConfiguration(versionedConfigStep); - configurationContext.setProperties(versionedConfigStep.getName(), stepConfig); + for (final Map.Entry> entry : migratedConfiguration.entrySet()) { + final StepConfiguration stepConfig = new StepConfiguration(new HashMap<>(entry.getValue())); + configurationContext.setProperties(entry.getKey(), stepConfig); } return configurationContext; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java index 7381302ff1d7..9f561d7b3a6c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorRepository.java @@ -36,6 +36,7 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.migration.ConnectorPropertyConfiguration; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.util.MockComponentLog; import org.junit.jupiter.api.Test; @@ -1968,4 +1969,186 @@ public void reset() { onStepConfiguredCalls.clear(); } } + + @Test + public void testInheritConfigurationMigratesActiveAndWorkingIndependently() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.forStep("step1").renameProperty("legacy", "renamed")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep activeStep = createVersionedStep("step1", + Map.of("legacy", createStringLiteralRef("active-value"))); + final VersionedConfigurationStep workingStep = createVersionedStep("step1", + Map.of("legacy", createStringLiteralRef("working-value"))); + + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(activeStep), List.of(workingStep), bundle); + + assertNotSame(node.getActiveFlowContext().getConfigurationContext(), + node.getWorkingFlowContext().getConfigurationContext()); + assertEquals("active-value", + node.getActiveFlowContext().getConfigurationContext().getProperty("step1", "renamed").getValue()); + assertEquals("working-value", + node.getWorkingFlowContext().getConfigurationContext().getProperty("step1", "renamed").getValue()); + assertFalse(node.getActiveFlowContext().getConfigurationContext().getPropertyNames("step1").contains("legacy")); + assertFalse(node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("step1").contains("legacy")); + } + + @Test + public void testInheritConfigurationInvokesMigratePropertiesWithEmptyStepLists() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.forStep("added-by-migration").setProperty("prop", "value")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(), List.of(), bundle); + + assertNotSame(node.getActiveFlowContext().getConfigurationContext(), + node.getWorkingFlowContext().getConfigurationContext()); + assertEquals("value", + node.getActiveFlowContext().getConfigurationContext().getProperty("added-by-migration", "prop").getValue()); + assertEquals("value", + node.getWorkingFlowContext().getConfigurationContext().getProperty("added-by-migration", "prop").getValue()); + } + + @Test + public void testMigratePropertiesRenamePropertyPropagatesToLiveConfiguration() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.forStep("step1").renameProperty("legacy", "renamed")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep step = createVersionedStep("step1", Map.of("legacy", createStringLiteralRef("kept-value"))); + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(step), List.of(step), bundle); + + final Set workingPropertyNames = node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("step1"); + assertTrue(workingPropertyNames.contains("renamed")); + assertFalse(workingPropertyNames.contains("legacy")); + assertEquals("kept-value", node.getWorkingFlowContext().getConfigurationContext().getProperty("step1", "renamed").getValue()); + + final Set activePropertyNames = node.getActiveFlowContext().getConfigurationContext().getPropertyNames("step1"); + assertTrue(activePropertyNames.contains("renamed")); + assertFalse(activePropertyNames.contains("legacy")); + } + + @Test + public void testMigratePropertiesRemovePropertyPropagatesToLiveConfiguration() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.forStep("step1").removeProperty("obsolete")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep step = createVersionedStep("step1", + Map.of("obsolete", createStringLiteralRef("gone"), "kept", createStringLiteralRef("value"))); + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(step), List.of(step), bundle); + + final Set propertyNames = node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("step1"); + assertFalse(propertyNames.contains("obsolete")); + assertTrue(propertyNames.contains("kept")); + } + + @Test + public void testMigratePropertiesAddPropertyPropagatesToLiveConfiguration() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.forStep("step1").setProperty("added", "new-value")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep step = createVersionedStep("step1", Map.of("original", createStringLiteralRef("v"))); + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(step), List.of(step), bundle); + + final Set propertyNames = node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("step1"); + assertTrue(propertyNames.contains("added")); + assertEquals("new-value", node.getWorkingFlowContext().getConfigurationContext().getProperty("step1", "added").getValue()); + } + + @Test + public void testMigratePropertiesRenameStepPreservesAllPropertiesAndFiresCallback() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.renameStep("old-step", "new-step")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep step = createVersionedStep("old-step", + Map.of("a", createStringLiteralRef("A"), "b", createStringLiteralRef("B"))); + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(step), List.of(step), bundle); + + assertTrue(node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("old-step").isEmpty()); + final Set newStepProperties = node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("new-step"); + assertEquals(Set.of("a", "b"), newStepProperties); + assertEquals("A", node.getWorkingFlowContext().getConfigurationContext().getProperty("new-step", "a").getValue()); + assertEquals("B", node.getWorkingFlowContext().getConfigurationContext().getProperty("new-step", "b").getValue()); + + assertTrue(connector.wasOnStepConfiguredCalled("new-step")); + } + + @Test + public void testMigratePropertiesRemoveStepDropsAllProperties() throws FlowUpdateException { + final MigratingConnector connector = new MigratingConnector(); + connector.setMigration(config -> config.removeStep("obsolete-step")); + final StandardConnectorNode node = createRealConnectorNode("connector-1", connector); + + final Bundle bundle = new Bundle(); + bundle.setGroup("org.apache.nifi"); + bundle.setArtifact("test-bundle"); + bundle.setVersion("1.0.0"); + + final VersionedConfigurationStep obsolete = createVersionedStep("obsolete-step", Map.of("a", createStringLiteralRef("A"))); + final VersionedConfigurationStep survivor = createVersionedStep("survivor", Map.of("b", createStringLiteralRef("B"))); + node.transitionStateForUpdating(); + node.prepareForUpdate(); + node.inheritConfiguration(List.of(obsolete, survivor), List.of(obsolete, survivor), bundle); + + assertTrue(node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("obsolete-step").isEmpty()); + assertEquals(Set.of("b"), node.getWorkingFlowContext().getConfigurationContext().getPropertyNames("survivor")); + } + + private static class MigratingConnector extends TrackingConnector { + private java.util.function.Consumer migration = config -> { }; + + public void setMigration(final java.util.function.Consumer migration) { + this.migration = migration; + } + + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + migration.accept(config); + } + } } diff --git a/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java new file mode 100644 index 000000000000..638b97005070 --- /dev/null +++ b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigVerificationResult.Outcome; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.ConnectorPropertyDescriptor; +import org.apache.nifi.components.connector.ConnectorPropertyGroup; +import org.apache.nifi.components.connector.PropertyType; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.migration.ConnectorPropertyConfiguration; +import org.apache.nifi.migration.ConnectorStepPropertyConfiguration; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.List; +import java.util.Map; + +/** + * Post-migration counterpart to the fixture Connector of the same simple type name in the + * {@code nifi-system-test-extensions} bundle. The pre-migration fixture defines a single + * {@code "Legacy Step"} with legacy property names; this class defines the migrated {@code "Kafka Connection"} step and + * renames / removes / adds properties in {@link #migrateProperties(ConnectorPropertyConfiguration)} so that a NAR swap + * followed by NiFi restart exercises the full {@code Connector.migrateProperties} code path end-to-end. + */ +public class MigratePropertiesConnector extends AbstractConnector { + + static final String LEGACY_STEP_NAME = "Legacy Step"; + static final String MIGRATED_STEP_NAME = "Kafka Connection"; + + static final String LEGACY_BROKER_URL_NAME = "legacy-broker-url"; + static final String LEGACY_CREDENTIALS_NAME = "legacy-credentials"; + static final String LEGACY_OBSOLETE_NAME = "legacy-obsolete"; + + static final String BROKER_URL_NAME = "Broker URL"; + static final String CREDENTIALS_NAME = "Credentials"; + static final String CLIENT_ID_NAME = "Client Id"; + static final String CLIENT_ID_MIGRATED_VALUE = "auto-migrated"; + + private static final ConnectorPropertyDescriptor BROKER_URL = new ConnectorPropertyDescriptor.Builder() + .name(BROKER_URL_NAME) + .description("Migrated string property.") + .required(false) + .type(PropertyType.STRING) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + private static final ConnectorPropertyDescriptor CREDENTIALS = new ConnectorPropertyDescriptor.Builder() + .name(CREDENTIALS_NAME) + .description("Migrated secret property.") + .required(false) + .type(PropertyType.SECRET) + .build(); + + private static final ConnectorPropertyDescriptor CLIENT_ID = new ConnectorPropertyDescriptor.Builder() + .name(CLIENT_ID_NAME) + .description("Added by migration when not already present.") + .required(false) + .type(PropertyType.STRING) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + private static final ConnectorPropertyGroup MIGRATED_PROPERTY_GROUP = new ConnectorPropertyGroup.Builder() + .name("Kafka Connection Properties") + .description("Migrated property group.") + .properties(List.of(BROKER_URL, CREDENTIALS, CLIENT_ID)) + .build(); + + private static final ConfigurationStep MIGRATED_STEP = new ConfigurationStep.Builder() + .name(MIGRATED_STEP_NAME) + .propertyGroups(List.of(MIGRATED_PROPERTY_GROUP)) + .build(); + + private final List configurationSteps = List.of(MIGRATED_STEP); + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup group = new VersionedProcessGroup(); + group.setName("Migrate Properties Flow"); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(group); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(new ConfigVerificationResult.Builder() + .outcome(Outcome.SUCCESSFUL) + .subject(stepName) + .verificationStepName("Migrate Properties Verification") + .explanation("Successful verification with properties: " + propertyValueOverrides) + .build()); + } + + @Override + public List getConfigurationSteps() { + return configurationSteps; + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } + + @Override + public void migrateProperties(final ConnectorPropertyConfiguration config) { + config.renameStep(LEGACY_STEP_NAME, MIGRATED_STEP_NAME); + final ConnectorStepPropertyConfiguration step = config.forStep(MIGRATED_STEP_NAME); + step.renameProperty(LEGACY_BROKER_URL_NAME, BROKER_URL_NAME); + step.renameProperty(LEGACY_CREDENTIALS_NAME, CREDENTIALS_NAME); + step.removeProperty(LEGACY_OBSOLETE_NAME); + if (!step.hasProperty(CLIENT_ID_NAME)) { + step.setProperty(CLIENT_ID_NAME, CLIENT_ID_MIGRATED_VALUE); + } + } +} diff --git a/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java new file mode 100644 index 000000000000..2af68ed125ce --- /dev/null +++ b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.tests.system; + +import org.apache.nifi.annotation.behavior.DynamicProperty; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.ParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Alternate-config-bundle copy of the pre-migration {@code PropertiesParameterProvider}. Preserving the same fully + * qualified class name here means that after the system-test NAR is swapped out for the alternate-config NAR, existing + * Parameter Provider instances (and any secret references pointing at their parameters) continue to resolve. Used by + * {@code ConnectorPropertyMigrationIT} so that a {@code SecretReference} carried through + * {@code Connector.migrateProperties} still resolves after the NAR swap. + */ +@DynamicProperty(name = "Parameter Group Name", value = "Parameters for the group", + expressionLanguageScope = ExpressionLanguageScope.NONE, + description = "Specifies parameters in a properties file format for the group") +public class PropertiesParameterProvider extends AbstractParameterProvider implements ParameterProvider { + + private static final PropertyDescriptor PARAMETERS = new PropertyDescriptor.Builder() + .name("parameters") + .displayName("Parameters") + .description("Specifies parameters in a properties file format") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + + @Override + protected List getSupportedPropertyDescriptors() { + return Collections.singletonList(PARAMETERS); + } + + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .dynamic(true) + .required(false) + .build(); + } + + @Override + public List fetchParameters(final ConfigurationContext context) { + final List groups = new ArrayList<>(); + + if (context.getProperty(PARAMETERS).isSet()) { + final List parameters = fetchParametersFromProperties(context.getProperty(PARAMETERS).getValue()); + groups.add(new ParameterGroup("Parameters", parameters)); + } + + for (final Map.Entry entry : context.getProperties().entrySet()) { + if (entry.getKey().isDynamic()) { + final String groupName = entry.getKey().getName(); + final List parameters = fetchParametersFromProperties(entry.getValue()); + groups.add(new ParameterGroup(groupName, parameters)); + } + } + + return groups; + } + + private List fetchParametersFromProperties(final String parametersPropertiesValue) { + final Properties parameters = new Properties(); + try { + parameters.load(new ByteArrayInputStream(parametersPropertiesValue.getBytes(StandardCharsets.UTF_8))); + } catch (final IOException e) { + throw new RuntimeException("Could not parse parameters as properties: " + parametersPropertiesValue); + } + return parameters.entrySet().stream() + .map(entry -> new Parameter.Builder() + .name(entry.getKey().toString()) + .value(entry.getValue().toString()) + .provided(true) + .build()) + .collect(Collectors.toList()); + } +} diff --git a/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector new file mode 100644 index 000000000000..cf74b8981c51 --- /dev/null +++ b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.connectors.tests.system.MigratePropertiesConnector diff --git a/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider new file mode 100644 index 000000000000..1cd3dd3f7cd3 --- /dev/null +++ b/nifi-system-tests/nifi-alternate-config-extensions-bundle/nifi-alternate-config-extensions/src/main/resources/META-INF/services/org.apache.nifi.parameter.ParameterProvider @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.parameter.tests.system.PropertiesParameterProvider diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java new file mode 100644 index 000000000000..d3bfbc11efa0 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/MigratePropertiesConnector.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigVerificationResult.Outcome; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.ConnectorPropertyDescriptor; +import org.apache.nifi.components.connector.ConnectorPropertyGroup; +import org.apache.nifi.components.connector.PropertyType; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.List; +import java.util.Map; + +/** + * Pre-migration fixture for the Connector migrateProperties system test. Defines a single + * {@link ConfigurationStep} named {@code "Legacy Step"} with legacy property names. Paired with the post-migration + * counterpart of the same simple type name in the alternate-config extensions bundle, which renames the step and + * properties, removes the obsolete property, and adds a new property via {@code migrateProperties}. + */ +public class MigratePropertiesConnector extends AbstractConnector { + + private static final ConnectorPropertyDescriptor LEGACY_BROKER_URL = new ConnectorPropertyDescriptor.Builder() + .name("legacy-broker-url") + .description("Legacy string property renamed by the post-migration connector.") + .required(false) + .type(PropertyType.STRING) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + private static final ConnectorPropertyDescriptor LEGACY_CREDENTIALS = new ConnectorPropertyDescriptor.Builder() + .name("legacy-credentials") + .description("Legacy secret property renamed by the post-migration connector.") + .required(false) + .type(PropertyType.SECRET) + .build(); + + private static final ConnectorPropertyDescriptor LEGACY_OBSOLETE = new ConnectorPropertyDescriptor.Builder() + .name("legacy-obsolete") + .description("Legacy property removed by the post-migration connector.") + .required(false) + .type(PropertyType.STRING) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + private static final ConnectorPropertyGroup LEGACY_PROPERTY_GROUP = new ConnectorPropertyGroup.Builder() + .name("Legacy Properties") + .description("Legacy properties migrated by the post-migration connector.") + .properties(List.of(LEGACY_BROKER_URL, LEGACY_CREDENTIALS, LEGACY_OBSOLETE)) + .build(); + + private static final ConfigurationStep LEGACY_STEP = new ConfigurationStep.Builder() + .name("Legacy Step") + .propertyGroups(List.of(LEGACY_PROPERTY_GROUP)) + .build(); + + private final List configurationSteps = List.of(LEGACY_STEP); + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public VersionedExternalFlow getInitialFlow() { + final VersionedProcessGroup group = new VersionedProcessGroup(); + group.setName("Migrate Properties Flow"); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(group); + return flow; + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + return getInitialFlow(); + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(new ConfigVerificationResult.Builder() + .outcome(Outcome.SUCCESSFUL) + .subject(stepName) + .verificationStepName("Migrate Properties Verification") + .explanation("Successful verification with properties: " + propertyValueOverrides) + .build()); + } + + @Override + public List getConfigurationSteps() { + return configurationSteps; + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) { + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector index a2d46427433a..632b1d138b22 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -19,6 +19,7 @@ org.apache.nifi.connectors.tests.system.CalculateConnector org.apache.nifi.connectors.tests.system.ComponentLifecycleConnector org.apache.nifi.connectors.tests.system.DataQueuingConnector org.apache.nifi.connectors.tests.system.GatedDataQueuingConnector +org.apache.nifi.connectors.tests.system.MigratePropertiesConnector org.apache.nifi.connectors.tests.system.NestedProcessGroupConnector org.apache.nifi.connectors.tests.system.NopConnector org.apache.nifi.connectors.tests.system.ParameterContextConnector diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorPropertyMigrationIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorPropertyMigrationIT.java new file mode 100644 index 000000000000..7e463a9e3f72 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorPropertyMigrationIT.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; +import org.apache.nifi.web.api.dto.ConnectorConfigurationDTO; +import org.apache.nifi.web.api.dto.ConnectorValueReferenceDTO; +import org.apache.nifi.web.api.dto.PropertyGroupConfigurationDTO; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.ParameterProviderEntity; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end system test for {@code Connector.migrateProperties}. A pre-migration + * {@code MigratePropertiesConnector} fixture in {@code nifi-system-test-extensions-nar} is created and configured + * through the REST API using legacy step / property names. NiFi is then stopped, the system-test extensions NARs are + * swapped for {@code nifi-alternate-config-extensions-nar} (which contains a same-simple-typed-name fixture that + * implements {@code migrateProperties}), and NiFi is restarted. The framework's {@code StandardConnectorRepository} + * sync path invokes {@code StandardConnectorNode.inheritConfiguration}, which drives + * {@code Connector.migrateProperties}. The test asserts that the persisted active and working configurations both + * reflect the migrated step and property names / values. + */ +public class ConnectorPropertyMigrationIT extends NiFiSystemIT { + + private static final String CONNECTOR_TYPE_SIMPLE_NAME = "MigratePropertiesConnector"; + + private static final String LEGACY_STEP_NAME = "Legacy Step"; + private static final String LEGACY_BROKER_URL_NAME = "legacy-broker-url"; + private static final String LEGACY_CREDENTIALS_NAME = "legacy-credentials"; + private static final String LEGACY_OBSOLETE_NAME = "legacy-obsolete"; + + private static final String MIGRATED_STEP_NAME = "Kafka Connection"; + private static final String BROKER_URL_NAME = "Broker URL"; + private static final String CREDENTIALS_NAME = "Credentials"; + private static final String CLIENT_ID_NAME = "Client Id"; + + private static final String BROKER_URL_VALUE = "kafka-broker:9092"; + private static final String OBSOLETE_VALUE = "gone"; + private static final String CLIENT_ID_MIGRATED_VALUE = "auto-migrated"; + + private static final String SECRET_NAME = "supersecret"; + private static final String FULLY_QUALIFIED_SECRET_NAME = "PropertiesParameterProvider.Parameters.supersecret"; + private static final String STRING_LITERAL_TYPE = "STRING_LITERAL"; + private static final String SECRET_REFERENCE_TYPE = "SECRET_REFERENCE"; + + @Override + protected boolean isAllowFactoryReuse() { + return false; + } + + @Override + protected boolean isDestroyEnvironmentAfterEachTest() { + return true; + } + + @AfterEach + public void restoreNars() { + // Stop the NiFi instance, ensure that the nifi-system-test-extensions-nar and nifi-alternate-config-extensions + // bundles are back where the next test expects them, and restart so the next test sees the default layout. + getNiFiInstance().stop(); + switchNarsBack(); + getNiFiInstance().start(true); + } + + @Test + public void testConnectorPropertyMigration() throws NiFiClientException, IOException, InterruptedException { + // Stand up a Parameter Provider with a single secret so that the legacy secret property can be configured with + // a SECRET_REFERENCE that survives the migration. + final ParameterProviderEntity paramProvider = getClientUtil().createParameterProvider("PropertiesParameterProvider"); + getClientUtil().updateParameterProviderProperties(paramProvider, Map.of("parameters", SECRET_NAME + "=" + SECRET_NAME)); + + // Create the pre-migration connector and configure the legacy step with a string literal, a secret reference, + // and an obsolete string literal that will be removed by migration. + final ConnectorEntity connector = getClientUtil().createConnector(CONNECTOR_TYPE_SIMPLE_NAME); + assertNotNull(connector); + + final ConnectorValueReferenceDTO brokerUrlRef = createStringLiteralReference(BROKER_URL_VALUE); + final ConnectorValueReferenceDTO credentialsRef = getClientUtil().createSecretValueReference(paramProvider.getId(), + SECRET_NAME, FULLY_QUALIFIED_SECRET_NAME); + final ConnectorValueReferenceDTO obsoleteRef = createStringLiteralReference(OBSOLETE_VALUE); + final Map legacyProperties = Map.of( + LEGACY_BROKER_URL_NAME, brokerUrlRef, + LEGACY_CREDENTIALS_NAME, credentialsRef, + LEGACY_OBSOLETE_NAME, obsoleteRef + ); + getClientUtil().configureConnectorWithReferences(connector.getId(), LEGACY_STEP_NAME, legacyProperties); + getClientUtil().applyConnectorUpdate(connector); + + // Sanity check: pre-restart state still has the legacy schema before we swap NARs. + final ConnectorEntity beforeRestart = getNifiClient().getConnectorClient().getConnector(connector.getId()); + final Map legacyActive = propertyValues(beforeRestart.getComponent().getActiveConfiguration(), LEGACY_STEP_NAME); + assertEquals(BROKER_URL_VALUE, legacyActive.get(LEGACY_BROKER_URL_NAME).getValue()); + assertEquals(SECRET_REFERENCE_TYPE, legacyActive.get(LEGACY_CREDENTIALS_NAME).getValueType()); + assertEquals(OBSOLETE_VALUE, legacyActive.get(LEGACY_OBSOLETE_NAME).getValue()); + + // Swap out the system-test-extensions NARs for the alternate-config NAR that defines the same simple-typed + // MigratePropertiesConnector with the migrated schema and a migrateProperties implementation. + getNiFiInstance().stop(); + switchOutNars(); + getNiFiInstance().start(true); + + // On restart, StandardConnectorRepository.syncConnector -> StandardConnectorNode.inheritConfiguration invokes + // Connector.migrateProperties for both the active and working configurations. Assert that each context now + // reflects the renamed step, renamed / removed / added properties, and that typed value references survive. + final ConnectorEntity afterRestart = getNifiClient().getConnectorClient().getConnector(connector.getId()); + assertMigratedConfiguration(afterRestart.getComponent().getActiveConfiguration(), "active"); + assertMigratedConfiguration(afterRestart.getComponent().getWorkingConfiguration(), "working"); + + getClientUtil().waitForValidConnector(connector.getId()); + } + + private void assertMigratedConfiguration(final ConnectorConfigurationDTO config, final String label) { + final List steps = config.getConfigurationStepConfigurations(); + assertNotNull(steps, label + " configuration is missing steps"); + assertTrue(steps.stream().noneMatch(step -> LEGACY_STEP_NAME.equals(step.getConfigurationStepName())), + label + " configuration still contains legacy step"); + + final Map migrated = propertyValues(config, MIGRATED_STEP_NAME); + + final ConnectorValueReferenceDTO brokerUrl = migrated.get(BROKER_URL_NAME); + assertNotNull(brokerUrl, label + " configuration is missing " + BROKER_URL_NAME); + assertEquals(STRING_LITERAL_TYPE, brokerUrl.getValueType()); + assertEquals(BROKER_URL_VALUE, brokerUrl.getValue()); + + final ConnectorValueReferenceDTO credentials = migrated.get(CREDENTIALS_NAME); + assertNotNull(credentials, label + " configuration is missing " + CREDENTIALS_NAME); + assertEquals(SECRET_REFERENCE_TYPE, credentials.getValueType()); + assertEquals(SECRET_NAME, credentials.getSecretName()); + + final ConnectorValueReferenceDTO clientId = migrated.get(CLIENT_ID_NAME); + assertNotNull(clientId, label + " configuration is missing " + CLIENT_ID_NAME); + assertEquals(STRING_LITERAL_TYPE, clientId.getValueType()); + assertEquals(CLIENT_ID_MIGRATED_VALUE, clientId.getValue()); + + assertFalse(migrated.containsKey(LEGACY_OBSOLETE_NAME), + label + " configuration still contains obsolete legacy property"); + assertFalse(migrated.containsKey(LEGACY_BROKER_URL_NAME), + label + " configuration still contains legacy broker URL name"); + assertFalse(migrated.containsKey(LEGACY_CREDENTIALS_NAME), + label + " configuration still contains legacy credentials name"); + } + + private Map propertyValues(final ConnectorConfigurationDTO config, final String stepName) { + final ConfigurationStepConfigurationDTO step = config.getConfigurationStepConfigurations().stream() + .filter(s -> stepName.equals(s.getConfigurationStepName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Configuration step not found: " + stepName)); + + final List groups = step.getPropertyGroupConfigurations(); + assertNotNull(groups, "No property groups on step " + stepName); + assertFalse(groups.isEmpty(), "No property groups on step " + stepName); + return groups.getFirst().getPropertyValues(); + } + + private ConnectorValueReferenceDTO createStringLiteralReference(final String value) { + final ConnectorValueReferenceDTO valueRef = new ConnectorValueReferenceDTO(); + valueRef.setValueType(STRING_LITERAL_TYPE); + valueRef.setValue(value); + return valueRef; + } + + private void switchOutNars() throws IOException { + final File instanceDir = getNiFiInstance().getInstanceDirectory(); + final File lib = new File(instanceDir, "lib"); + final File alternateConfig = new File(lib, "alternate-config"); + + moveNars(lib, "nifi-system-test-extensions-nar-.*", alternateConfig); + moveNars(lib, "nifi-system-test-extensions-services-nar-.*", alternateConfig); + moveNars(lib, "nifi-system-test-extensions-services-api-nar-.*", alternateConfig); + moveNars(alternateConfig, "nifi-alternate-config.*", lib); + + final File workDir = new File(instanceDir, "work/nar/extensions"); + deleteRecursively(workDir); + } + + private void switchNarsBack() { + final File instanceDir = getNiFiInstance().getInstanceDirectory(); + final File lib = new File(instanceDir, "lib"); + final File alternateConfig = new File(lib, "alternate-config"); + + moveNars(alternateConfig, "nifi-system-test-extensions-nar-.*", lib); + moveNars(alternateConfig, "nifi-system-test-extensions-services-nar-.*", lib); + moveNars(alternateConfig, "nifi-system-test-extensions-services-api-nar-.*", lib); + moveNars(lib, "nifi-alternate-config.*", alternateConfig); + } + + private void deleteRecursively(final File file) throws IOException { + Files.walkFileTree(file.toPath(), new FileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path visited, final BasicFileAttributes attrs) throws IOException { + Files.delete(visited); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(final Path visited, final IOException exc) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + private File findFile(final File dir, final String regex) { + final Pattern pattern = Pattern.compile(regex); + final File[] files = dir.listFiles(file -> pattern.matcher(file.getName()).find()); + if (files == null || files.length != 1) { + return null; + } + return files[0]; + } + + private void moveNars(final File source, final String regex, final File target) { + final File libNar = findFile(source, regex); + assertNotNull(libNar); + final File libNarTarget = new File(target, libNar.getName()); + assertTrue(libNar.renameTo(libNarTarget)); + } +}