From c2d785eeb67fa6dda758776dfac89e328163ae69 Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Fri, 11 Sep 2026 13:14:41 +0530 Subject: [PATCH 1/2] Fixes #28299: mask and encrypt secrets nested in a schema oneOf jsonschema2pojo ignores `oneOf`, so a `oneOf` property is generated as a bare `Object` and holds a `LinkedHashMap` at runtime. Both `PasswordEntityMasker` and `SecretsManager` recurse only into values whose package starts with `org.openmetadata`, so that map is skipped entirely: every `format: password` leaf below it was never Fernet-encrypted, never handed to the external secrets manager, and never masked on read. The UI lost the `format: password` marker for the same reason and rendered the value in a plain text input, which is what #28299 reports for SFTP. Audited every connection schema: 41 (connection, property) pairs were uncovered. 37 are fixed here by registering a converter that re-types the property before either walk runs -- SFTP, OpenSearch, Alation, Databricks Pipeline, REST, PowerBI, Microsoft Access, MyDb, QuestDB auth configs, the ES/OS SSL certificate values, and every `sslConfig` (its certificate and key fields are `format: password` too). `NestedConfigClassConverter` covers the 26 connections that need nothing but re-typing, so the registry stays the single source of truth instead of 26 near-identical classes. SFTP's auth branches were inline `#/definitions`, and jsonschema2pojo generates a class per schema *file*, so no class existed to convert into; they are extracted to `drive/sftp/{basicAuth,keyAuth}.json` with the titles unchanged. The stray `type: object` on `common/sslConfig.json` is dropped for the same reason. `NestedSecretConverterCoverageTest` is the regression gate: it walks every connection schema on the classpath, builds a payload per secret-bearing `oneOf` branch, runs the registered converter and fails if the property is still a `Map`. Its `KNOWN_UNCONVERTIBLE` allowlist holds the four remaining schema-extraction cases so they stay visible and the set cannot grow. Co-Authored-By: Claude Opus 5 (1M context) --- .../ingestion/source/drive/sftp/connection.py | 12 +- .../tests/unit/topology/drive/test_sftp.py | 32 +- .../CassandraConnectionClassConverter.java | 5 + .../secrets/converter/ClassConverter.java | 32 ++ .../converter/ClassConverterFactory.java | 327 ++++++++++++++---- ...ElasticSearchConnectionClassConverter.java | 12 + .../HiveConnectionClassConverter.java | 5 + .../converter/NestedConfigClassConverter.java | 43 +++ .../OpenSearchConnectionClassConverter.java | 50 +++ .../PrefectConnectionClassConverter.java | 5 + .../TableauConnectionClassConverter.java | 5 + .../service/secrets/DBSecretsManagerTest.java | 22 ++ .../NestedSecretConverterCoverageTest.java | 294 ++++++++++++++++ .../masker/PasswordEntityMaskerTest.java | 52 +++ .../connections/common/sslConfig.json | 1 - .../connections/drive/sftp/basicAuth.json | 23 ++ .../connections/drive/sftp/keyAuth.json | 30 ++ .../connections/drive/sftpConnection.json | 51 +-- .../connections/common/sslConfig.json | 1 - .../connections/drive/sftp/basicAuth.json | 25 ++ .../connections/drive/sftp/keyAuth.json | 32 ++ .../connections/drive/sftpConnection.json | 57 +-- .../search/elasticSearchConnection.json | 1 - .../search/openSearchConnection.json | 1 - .../connections/serviceConnection.json | 118 +------ .../ingestionSchemas/testSuitePipeline.json | 109 +++--- .../src/jsons/ingestionSchemas/workflow.json | 109 +++--- .../ServiceConnectionDetailsUtils.test.tsx | 157 +++++++++ .../utils/ServiceConnectionDetailsUtils.tsx | 115 +++++- 29 files changed, 1302 insertions(+), 424 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NestedConfigClassConverter.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/OpenSearchConnectionClassConverter.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/NestedSecretConverterCoverageTest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/basicAuth.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/keyAuth.json create mode 100644 openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/basicAuth.json create mode 100644 openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/keyAuth.json create mode 100644 openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.test.tsx diff --git a/ingestion/src/metadata/ingestion/source/drive/sftp/connection.py b/ingestion/src/metadata/ingestion/source/drive/sftp/connection.py index 42d8565ab42a..532bebda95f9 100644 --- a/ingestion/src/metadata/ingestion/source/drive/sftp/connection.py +++ b/ingestion/src/metadata/ingestion/source/drive/sftp/connection.py @@ -22,9 +22,11 @@ from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) -from metadata.generated.schema.entity.services.connections.drive.sftpConnection import ( - BasicAuth, - KeyAuth, +from metadata.generated.schema.entity.services.connections.drive.sftp.basicAuth import ( + UsernamePasswordAuthentication, +) +from metadata.generated.schema.entity.services.connections.drive.sftp.keyAuth import ( + PrivateKeyAuthentication, ) from metadata.generated.schema.entity.services.connections.drive.sftpConnection import ( SftpConnection as SftpConnectionConfig, @@ -98,13 +100,13 @@ def _get_client(self) -> SftpClient: auth_type = connection.authType - if isinstance(auth_type, BasicAuth): + if isinstance(auth_type, UsernamePasswordAuthentication): password = auth_type.password.get_secret_value() if auth_type.password else None transport.connect( username=auth_type.username, password=password, ) - elif isinstance(auth_type, KeyAuth): + elif isinstance(auth_type, PrivateKeyAuthentication): private_key_str = auth_type.privateKey.get_secret_value() passphrase = ( auth_type.privateKeyPassphrase.get_secret_value() if auth_type.privateKeyPassphrase else None diff --git a/ingestion/tests/unit/topology/drive/test_sftp.py b/ingestion/tests/unit/topology/drive/test_sftp.py index c91d59cc93d9..9674c4805da7 100644 --- a/ingestion/tests/unit/topology/drive/test_sftp.py +++ b/ingestion/tests/unit/topology/drive/test_sftp.py @@ -19,9 +19,13 @@ import pytest +from metadata.generated.schema.entity.services.connections.drive.sftp.basicAuth import ( + UsernamePasswordAuthentication, +) +from metadata.generated.schema.entity.services.connections.drive.sftp.keyAuth import ( + PrivateKeyAuthentication, +) from metadata.generated.schema.entity.services.connections.drive.sftpConnection import ( - BasicAuth, - KeyAuth, SftpConnection, ) from metadata.generated.schema.metadataIngestion.workflow import ( @@ -209,14 +213,14 @@ class TestSftpConnection(TestCase): """Test SFTP connection configuration""" def test_basic_auth_config(self): - """Test BasicAuth configuration""" - auth = BasicAuth(username="testuser", password="testpass") + """Test username/password auth configuration""" + auth = UsernamePasswordAuthentication(username="testuser", password="testpass") self.assertEqual(auth.username, "testuser") self.assertEqual(auth.password.get_secret_value(), "testpass") def test_key_auth_config(self): - """Test KeyAuth configuration""" - auth = KeyAuth( + """Test private-key auth configuration""" + auth = PrivateKeyAuthentication( username="testuser", privateKey="-----BEGIN RSA PRIVATE KEY-----\ntest\n-----END RSA PRIVATE KEY-----", privateKeyPassphrase="passphrase", @@ -230,7 +234,7 @@ def test_sftp_connection_config(self): config = SftpConnection( host="localhost", port=22, - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), rootDirectories=["/data", "/home"], ) self.assertEqual(config.host, "localhost") @@ -383,7 +387,7 @@ def test_get_connection_basic_auth(self, mock_sftp_client, mock_transport): connection = SftpConnection( host="localhost", port=22, - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), ) client = SftpConnectionHandler(connection)._get_client() @@ -412,7 +416,7 @@ def test_get_connection_key_auth(self, mock_sftp_client, mock_transport, mock_pa connection = SftpConnection( host="localhost", port=2222, - authType=KeyAuth( + authType=PrivateKeyAuthentication( username="user", privateKey="-----BEGIN RSA PRIVATE KEY-----\ntest\n-----END RSA PRIVATE KEY-----", ), @@ -589,7 +593,7 @@ def test_structured_data_files_only_default(self): """Test structuredDataFilesOnly defaults to False""" connection = SftpConnection( host="localhost", - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), ) self.assertFalse(connection.structuredDataFilesOnly) @@ -597,7 +601,7 @@ def test_structured_data_files_only_enabled(self): """Test structuredDataFilesOnly can be enabled""" connection = SftpConnection( host="localhost", - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), structuredDataFilesOnly=True, ) self.assertTrue(connection.structuredDataFilesOnly) @@ -606,7 +610,7 @@ def test_extract_sample_data_default(self): """Test extractSampleData defaults to False""" connection = SftpConnection( host="localhost", - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), ) self.assertFalse(connection.extractSampleData) @@ -614,7 +618,7 @@ def test_extract_sample_data_enabled(self): """Test extractSampleData can be enabled""" connection = SftpConnection( host="localhost", - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), extractSampleData=True, ) self.assertTrue(connection.extractSampleData) @@ -623,7 +627,7 @@ def test_both_options_enabled(self): """Test both options can be enabled together""" connection = SftpConnection( host="localhost", - authType=BasicAuth(username="user", password="pass"), + authType=UsernamePasswordAuthentication(username="user", password="pass"), structuredDataFilesOnly=True, extractSampleData=True, ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CassandraConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CassandraConnectionClassConverter.java index b17e067783ad..01ec36f99796 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CassandraConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/CassandraConnectionClassConverter.java @@ -14,6 +14,7 @@ package org.openmetadata.service.secrets.converter; import java.util.List; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; import org.openmetadata.schema.services.connections.database.CassandraConnection; import org.openmetadata.schema.services.connections.database.cassandra.CloudConfig; import org.openmetadata.schema.services.connections.database.common.basicAuth; @@ -27,6 +28,8 @@ public class CassandraConnectionClassConverter extends ClassConverter { private static final List> CONFIG_SOURCE_CLASSES = List.of(basicAuth.class, CloudConfig.class); + private static final List> SSL_SOURCE_CLASSES = List.of(ValidateSSLClientConfig.class); + public CassandraConnectionClassConverter() { super(CassandraConnection.class); } @@ -39,6 +42,8 @@ public Object convert(Object object) { tryToConvert(cassandraConnection.getAuthType(), CONFIG_SOURCE_CLASSES) .ifPresent(cassandraConnection::setAuthType); + convertProperty(cassandraConnection, "sslConfig", SSL_SOURCE_CLASSES); + return cassandraConnection; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverter.java index b3d627c6c84e..41319c6aeef8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverter.java @@ -13,10 +13,12 @@ package org.openmetadata.service.secrets.converter; +import java.lang.reflect.Method; import java.util.List; import java.util.Objects; import java.util.Optional; import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.exception.ReflectionException; /** * Currently when an object is converted into a specific class using `JsonUtils.convertValue` there`Object` fields that @@ -67,6 +69,36 @@ protected Optional tryToConvertOrFail(Object object, List> cand } // method called when and Object field can expect a HashMap or a specific class + /** + * Reads {@code property} off {@code target}, converts it against {@code candidates} and writes it + * back. + * + *

Only for {@code Object}-typed properties, i.e. the ones jsonschema2pojo emits for a JSON + * Schema {@code oneOf}. Jackson leaves a {@code LinkedHashMap} in such a field, and neither the + * password masker nor the secrets manager descends into a non-{@code org.openmetadata} value, so + * every {@code format: password} leaf below it stays in the clear until it is typed again here. + */ + protected void convertProperty(Object target, String property, List> candidates) { + String accessorSuffix = Character.toUpperCase(property.charAt(0)) + property.substring(1); + try { + Method getter = target.getClass().getMethod("get" + accessorSuffix); + tryToConvert(getter.invoke(target), candidates) + .ifPresent( + value -> { + try { + target + .getClass() + .getMethod("set" + accessorSuffix, Object.class) + .invoke(target, value); + } catch (ReflectiveOperationException e) { + throw new ReflectionException(e.getMessage()); + } + }); + } catch (ReflectiveOperationException e) { + throw new ReflectionException(e.getMessage()); + } + } + protected Optional tryToConvert(Object object, List> candidateClasses) { if (object != null) { Optional converted = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index f0b75d69dc06..e2072c1166c4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -13,6 +13,8 @@ package org.openmetadata.service.secrets.converter; +import java.util.HashMap; +import java.util.List; import java.util.Map; import lombok.Getter; import org.openmetadata.schema.auth.SSOAuthMechanism; @@ -21,11 +23,27 @@ import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.metadataIngestion.DbtPipeline; import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig; +import org.openmetadata.schema.security.credentials.ApiAccessTokenAuth; +import org.openmetadata.schema.security.credentials.BasicAuth; import org.openmetadata.schema.security.credentials.GCPCredentials; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; +import org.openmetadata.schema.services.common.SSLCertPaths; +import org.openmetadata.schema.services.common.SSLCertValues; +import org.openmetadata.schema.services.common.SSLConfig; +import org.openmetadata.schema.services.connections.api.OpenAPISchemaFilePath; +import org.openmetadata.schema.services.connections.api.OpenAPISchemaS3; +import org.openmetadata.schema.services.connections.api.OpenAPISchemaURL; +import org.openmetadata.schema.services.connections.api.RestConnection; import org.openmetadata.schema.services.connections.dashboard.LookerConnection; +import org.openmetadata.schema.services.connections.dashboard.OmniConnection; +import org.openmetadata.schema.services.connections.dashboard.PowerBIConnection; +import org.openmetadata.schema.services.connections.dashboard.SapS4HanaConnection; +import org.openmetadata.schema.services.connections.dashboard.SsrsConnection; import org.openmetadata.schema.services.connections.dashboard.SupersetConnection; import org.openmetadata.schema.services.connections.dashboard.TableauConnection; import org.openmetadata.schema.services.connections.dashboard.ThoughtSpotConnection; +import org.openmetadata.schema.services.connections.dashboard.powerbi.AzureConfig; +import org.openmetadata.schema.services.connections.dashboard.powerbi.S3Config; import org.openmetadata.schema.services.connections.database.BigQueryConnection; import org.openmetadata.schema.services.connections.database.BigTableConnection; import org.openmetadata.schema.services.connections.database.CassandraConnection; @@ -33,29 +51,52 @@ import org.openmetadata.schema.services.connections.database.CockroachConnection; import org.openmetadata.schema.services.connections.database.DatabricksConnection; import org.openmetadata.schema.services.connections.database.DatalakeConnection; +import org.openmetadata.schema.services.connections.database.Db2Connection; import org.openmetadata.schema.services.connections.database.DeltaLakeConnection; +import org.openmetadata.schema.services.connections.database.DorisConnection; import org.openmetadata.schema.services.connections.database.DremioConnection; import org.openmetadata.schema.services.connections.database.GreenplumConnection; import org.openmetadata.schema.services.connections.database.HiveConnection; +import org.openmetadata.schema.services.connections.database.InformixConnection; import org.openmetadata.schema.services.connections.database.MicrosoftAccessConnection; +import org.openmetadata.schema.services.connections.database.MongoDBConnection; import org.openmetadata.schema.services.connections.database.MssqlConnection; +import org.openmetadata.schema.services.connections.database.MyDbConnection; import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.PostgresConnection; +import org.openmetadata.schema.services.connections.database.QuestDBConnection; import org.openmetadata.schema.services.connections.database.RedshiftConnection; import org.openmetadata.schema.services.connections.database.SalesforceConnection; +import org.openmetadata.schema.services.connections.database.SapErpConnection; import org.openmetadata.schema.services.connections.database.SapHanaConnection; +import org.openmetadata.schema.services.connections.database.SapSuccessFactorsConnection; import org.openmetadata.schema.services.connections.database.StarRocksConnection; import org.openmetadata.schema.services.connections.database.TimescaleConnection; import org.openmetadata.schema.services.connections.database.TrinoConnection; import org.openmetadata.schema.services.connections.database.UnityCatalogConnection; +import org.openmetadata.schema.services.connections.database.common.basicAuth; +import org.openmetadata.schema.services.connections.database.databricks.AzureADSetup; +import org.openmetadata.schema.services.connections.database.databricks.DatabricksOAuth; +import org.openmetadata.schema.services.connections.database.databricks.PersonalAccessToken; import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.database.deltalake.StorageConfig; import org.openmetadata.schema.services.connections.drive.GoogleDriveConnection; +import org.openmetadata.schema.services.connections.drive.SftpConnection; +import org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth; +import org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth; +import org.openmetadata.schema.services.connections.messaging.KafkaConnection; +import org.openmetadata.schema.services.connections.messaging.NatsConnection; import org.openmetadata.schema.services.connections.messaging.PubSubConnection; +import org.openmetadata.schema.services.connections.metadata.AlationConnection; +import org.openmetadata.schema.services.connections.metadata.AlationSinkConnection; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection; import org.openmetadata.schema.services.connections.pipeline.AirbyteConnection; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; import org.openmetadata.schema.services.connections.pipeline.AirflowRestApiConnection; +import org.openmetadata.schema.services.connections.pipeline.DatabricksPipelineConnection; +import org.openmetadata.schema.services.connections.pipeline.FivetranConnection; +import org.openmetadata.schema.services.connections.pipeline.FlinkConnection; import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; import org.openmetadata.schema.services.connections.pipeline.MulesoftConnection; import org.openmetadata.schema.services.connections.pipeline.NifiConnection; @@ -63,9 +104,13 @@ import org.openmetadata.schema.services.connections.pipeline.PrefectConnection; import org.openmetadata.schema.services.connections.pipeline.SSISConnection; import org.openmetadata.schema.services.connections.pipeline.WherescapeConnection; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth; +import org.openmetadata.schema.services.connections.pipeline.openlineage.KafkaBrokerConfig; import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; +import org.openmetadata.schema.services.connections.search.OpenSearchConnection; import org.openmetadata.schema.services.connections.security.RangerConnection; import org.openmetadata.schema.services.connections.storage.GCSConnection; +import org.openmetadata.schema.services.connections.storage.S3Connection; /** Factory class to get a `ClassConverter` based on the service class. */ public final class ClassConverterFactory { @@ -75,65 +120,231 @@ private ClassConverterFactory() { @Getter private static final Map, ClassConverter> converterMap; + /** + * Connections whose {@code Object} properties -- the ones a JSON Schema {@code oneOf} produces -- + * only need to be re-typed so the password masker and the secrets manager can walk into them. See + * {@link NestedConfigClassConverter}; anything needing more logic gets its own converter above. + */ + private static final Map, ClassConverter> NESTED_CONFIG_CONVERTERS = + Map.ofEntries( + Map.entry( + AlationConnection.class, + new NestedConfigClassConverter( + AlationConnection.class, + Map.of( + "authType", List.of(BasicAuth.class, ApiAccessTokenAuth.class), + "connection", List.of(PostgresConnection.class, MysqlConnection.class)))), + Map.entry( + AlationSinkConnection.class, + new NestedConfigClassConverter( + AlationSinkConnection.class, + Map.of( + "authType", List.of(BasicAuth.class, ApiAccessTokenAuth.class), + "sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + DatabricksPipelineConnection.class, + new NestedConfigClassConverter( + DatabricksPipelineConnection.class, + Map.of( + "authType", + List.of( + PersonalAccessToken.class, DatabricksOAuth.class, AzureADSetup.class)))), + Map.entry( + Db2Connection.class, + new NestedConfigClassConverter( + Db2Connection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + DorisConnection.class, + new NestedConfigClassConverter( + DorisConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + FivetranConnection.class, + new NestedConfigClassConverter( + FivetranConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + FlinkConnection.class, + new NestedConfigClassConverter( + FlinkConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + InformixConnection.class, + new NestedConfigClassConverter( + InformixConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + KafkaBrokerConfig.class, + new NestedConfigClassConverter( + KafkaBrokerConfig.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + KafkaConnection.class, + new NestedConfigClassConverter( + KafkaConnection.class, + Map.of( + "consumerConfigSSL", List.of(ValidateSSLClientConfig.class), + "schemaRegistrySSL", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + MatillionETLAuth.class, + new NestedConfigClassConverter( + MatillionETLAuth.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + MicrosoftAccessConnection.class, + new NestedConfigClassConverter( + MicrosoftAccessConnection.class, + Map.of("connection", List.of(S3Connection.class)))), + Map.entry( + MongoDBConnection.class, + new NestedConfigClassConverter( + MongoDBConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + MyDbConnection.class, + new NestedConfigClassConverter( + MyDbConnection.class, + Map.of( + "authType", List.of(basicAuth.class), + "sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + NatsConnection.class, + new NestedConfigClassConverter( + NatsConnection.class, + Map.of("tlsConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + OmniConnection.class, + new NestedConfigClassConverter( + OmniConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + OpenMetadataConnection.class, + new NestedConfigClassConverter( + OpenMetadataConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + PowerBIConnection.class, + new NestedConfigClassConverter( + PowerBIConnection.class, + Map.of( + "pbitFilesSource", + List.of( + AzureConfig.class, + org.openmetadata.schema.services.connections.dasboard.powerbi.GCSConfig + .class, + S3Config.class)))), + Map.entry( + QuestDBConnection.class, + new NestedConfigClassConverter( + QuestDBConnection.class, Map.of("authType", List.of(basicAuth.class)))), + Map.entry( + RestConnection.class, + new NestedConfigClassConverter( + RestConnection.class, + Map.of( + "openAPISchemaConnection", + List.of( + OpenAPISchemaURL.class, + OpenAPISchemaFilePath.class, + OpenAPISchemaS3.class), + "sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + SSLConfig.class, + new NestedConfigClassConverter( + SSLConfig.class, + Map.of("certificates", List.of(SSLCertPaths.class, SSLCertValues.class)))), + Map.entry( + SapErpConnection.class, + new NestedConfigClassConverter( + SapErpConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + SapS4HanaConnection.class, + new NestedConfigClassConverter( + SapS4HanaConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + SapSuccessFactorsConnection.class, + new NestedConfigClassConverter( + SapSuccessFactorsConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class)))), + Map.entry( + SftpConnection.class, + new NestedConfigClassConverter( + SftpConnection.class, + Map.of("authType", List.of(SftpBasicAuth.class, SftpKeyAuth.class)))), + Map.entry( + SsrsConnection.class, + new NestedConfigClassConverter( + SsrsConnection.class, + Map.of("sslConfig", List.of(ValidateSSLClientConfig.class))))); + static { - converterMap = - Map.ofEntries( - Map.entry(AirbyteConnection.class, new AirbyteConnectionClassConverter()), - Map.entry(AirflowConnection.class, new AirflowConnectionClassConverter()), - Map.entry(AirflowRestApiConnection.class, new AirflowRestApiConnectionClassConverter()), - Map.entry(BigQueryConnection.class, new BigQueryConnectionClassConverter()), - Map.entry(BigTableConnection.class, new BigTableConnectionClassConverter()), - Map.entry(DatalakeConnection.class, new DatalakeConnectionClassConverter()), - Map.entry(DeltaLakeConnection.class, new DeltaLakeConnectionClassConverter()), - Map.entry(DremioConnection.class, new DremioConnectionClassConverter()), - Map.entry(DbtGCSConfig.class, new DbtGCSConfigClassConverter()), - Map.entry(DbtPipeline.class, new DbtPipelineClassConverter()), - Map.entry(ElasticSearchConnection.class, new ElasticSearchConnectionClassConverter()), - Map.entry(GCSConfig.class, new GCPConfigClassConverter()), - Map.entry(GCPCredentials.class, new GcpCredentialsClassConverter()), - Map.entry(GCSConnection.class, new GcpConnectionClassConverter()), - Map.entry(GoogleDriveConnection.class, new GoogleDriveConnectionClassConverter()), - Map.entry(PubSubConnection.class, new PubSubConnectionClassConverter()), - Map.entry(HiveConnection.class, new HiveConnectionClassConverter()), - Map.entry(LookerConnection.class, new LookerConnectionClassConverter()), - Map.entry( - MicrosoftAccessConnection.class, new MicrosoftAccessConnectionClassConverter()), - Map.entry(MssqlConnection.class, new MssqlConnectionClassConverter()), - Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), - Map.entry(RedshiftConnection.class, new RedshiftConnectionClassConverter()), - Map.entry(GreenplumConnection.class, new GreenplumConnectionClassConverter()), - Map.entry(PostgresConnection.class, new PostgresConnectionClassConverter()), - Map.entry(SapHanaConnection.class, new SapHanaConnectionClassConverter()), - Map.entry(StarRocksConnection.class, new StarRocksConnectionClassConverter()), - Map.entry(StorageConfig.class, new StorageConfigClassConverter()), - Map.entry(SupersetConnection.class, new SupersetConnectionClassConverter()), - Map.entry(SSOAuthMechanism.class, new SSOAuthMechanismClassConverter()), - Map.entry(TableauConnection.class, new TableauConnectionClassConverter()), - Map.entry(ThoughtSpotConnection.class, new ThoughtSpotConnectionClassConverter()), - Map.entry(MulesoftConnection.class, new MulesoftConnectionClassConverter()), - Map.entry(SalesforceConnection.class, new SalesforceConnectorClassConverter()), - Map.entry( - TestServiceConnectionRequest.class, - new TestServiceConnectionRequestClassConverter()), - Map.entry( - TestSparkEngineConnectionRequest.class, - new TestSparkEngineConnectionRequestClassConverter()), - Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), - Map.entry(Workflow.class, new WorkflowClassConverter()), - Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter()), - Map.entry(ClickzettaConnection.class, new ClickzettaConnectionClassConverter()), - Map.entry(NifiConnection.class, new NifiConnectionClassConverter()), - Map.entry(OpenLineageConnection.class, new OpenLineageConnectionClassConverter()), - Map.entry(MatillionConnection.class, new MatillionConnectionClassConverter()), - Map.entry(PrefectConnection.class, new PrefectConnectionClassConverter()), - Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter()), - Map.entry(RangerConnection.class, new RangerConnectionClassConverter()), - Map.entry(DatabricksConnection.class, new DatabricksConnectionClassConverter()), - Map.entry(UnityCatalogConnection.class, new UnityCatalogConnectionClassConverter()), - Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()), - Map.entry(SSISConnection.class, new SsisConnectionClassConverter()), - Map.entry(WherescapeConnection.class, new WherescapeConnectionClassConverter()), - Map.entry(TimescaleConnection.class, new TimescaleConnectionClassConverter())); + Map, ClassConverter> converters = + new HashMap<>( + Map.ofEntries( + Map.entry(AirbyteConnection.class, new AirbyteConnectionClassConverter()), + Map.entry(AirflowConnection.class, new AirflowConnectionClassConverter()), + Map.entry( + AirflowRestApiConnection.class, new AirflowRestApiConnectionClassConverter()), + Map.entry(BigQueryConnection.class, new BigQueryConnectionClassConverter()), + Map.entry(BigTableConnection.class, new BigTableConnectionClassConverter()), + Map.entry(DatalakeConnection.class, new DatalakeConnectionClassConverter()), + Map.entry(DeltaLakeConnection.class, new DeltaLakeConnectionClassConverter()), + Map.entry(DremioConnection.class, new DremioConnectionClassConverter()), + Map.entry(DbtGCSConfig.class, new DbtGCSConfigClassConverter()), + Map.entry(DbtPipeline.class, new DbtPipelineClassConverter()), + Map.entry( + ElasticSearchConnection.class, new ElasticSearchConnectionClassConverter()), + Map.entry(OpenSearchConnection.class, new OpenSearchConnectionClassConverter()), + Map.entry(GCSConfig.class, new GCPConfigClassConverter()), + Map.entry(GCPCredentials.class, new GcpCredentialsClassConverter()), + Map.entry(GCSConnection.class, new GcpConnectionClassConverter()), + Map.entry(GoogleDriveConnection.class, new GoogleDriveConnectionClassConverter()), + Map.entry(PubSubConnection.class, new PubSubConnectionClassConverter()), + Map.entry(HiveConnection.class, new HiveConnectionClassConverter()), + Map.entry(LookerConnection.class, new LookerConnectionClassConverter()), + Map.entry( + MicrosoftAccessConnection.class, new MicrosoftAccessConnectionClassConverter()), + Map.entry(MssqlConnection.class, new MssqlConnectionClassConverter()), + Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), + Map.entry(RedshiftConnection.class, new RedshiftConnectionClassConverter()), + Map.entry(GreenplumConnection.class, new GreenplumConnectionClassConverter()), + Map.entry(PostgresConnection.class, new PostgresConnectionClassConverter()), + Map.entry(SapHanaConnection.class, new SapHanaConnectionClassConverter()), + Map.entry(StarRocksConnection.class, new StarRocksConnectionClassConverter()), + Map.entry(StorageConfig.class, new StorageConfigClassConverter()), + Map.entry(SupersetConnection.class, new SupersetConnectionClassConverter()), + Map.entry(SSOAuthMechanism.class, new SSOAuthMechanismClassConverter()), + Map.entry(TableauConnection.class, new TableauConnectionClassConverter()), + Map.entry(ThoughtSpotConnection.class, new ThoughtSpotConnectionClassConverter()), + Map.entry(MulesoftConnection.class, new MulesoftConnectionClassConverter()), + Map.entry(SalesforceConnection.class, new SalesforceConnectorClassConverter()), + Map.entry( + TestServiceConnectionRequest.class, + new TestServiceConnectionRequestClassConverter()), + Map.entry( + TestSparkEngineConnectionRequest.class, + new TestSparkEngineConnectionRequestClassConverter()), + Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), + Map.entry(Workflow.class, new WorkflowClassConverter()), + Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter()), + Map.entry(ClickzettaConnection.class, new ClickzettaConnectionClassConverter()), + Map.entry(NifiConnection.class, new NifiConnectionClassConverter()), + Map.entry(OpenLineageConnection.class, new OpenLineageConnectionClassConverter()), + Map.entry(MatillionConnection.class, new MatillionConnectionClassConverter()), + Map.entry(PrefectConnection.class, new PrefectConnectionClassConverter()), + Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter()), + Map.entry(RangerConnection.class, new RangerConnectionClassConverter()), + Map.entry(DatabricksConnection.class, new DatabricksConnectionClassConverter()), + Map.entry(UnityCatalogConnection.class, new UnityCatalogConnectionClassConverter()), + Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()), + Map.entry(SSISConnection.class, new SsisConnectionClassConverter()), + Map.entry(WherescapeConnection.class, new WherescapeConnectionClassConverter()), + Map.entry(TimescaleConnection.class, new TimescaleConnectionClassConverter()))); + converters.putAll(NESTED_CONFIG_CONVERTERS); + converterMap = Map.copyOf(converters); } public static ClassConverter getConverter(Class clazz) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ElasticSearchConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ElasticSearchConnectionClassConverter.java index 573dbac71c69..3f0740bd22ac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ElasticSearchConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ElasticSearchConnectionClassConverter.java @@ -14,6 +14,8 @@ package org.openmetadata.service.secrets.converter; import java.util.List; +import org.openmetadata.schema.services.common.SSLCertPaths; +import org.openmetadata.schema.services.common.SSLCertValues; import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; import org.openmetadata.schema.services.connections.search.elasticSearch.ESAPIAuth; import org.openmetadata.schema.services.connections.search.elasticSearch.ESBasicAuth; @@ -25,6 +27,10 @@ public class ElasticSearchConnectionClassConverter extends ClassConverter { private static final List> CONFIG_SOURCE_CLASSES = List.of(ESBasicAuth.class, ESAPIAuth.class); + static final String CERTIFICATES = "certificates"; + static final List> SSL_CERTIFICATE_CLASSES = + List.of(SSLCertPaths.class, SSLCertValues.class); + // public ElasticSearchConnectionClassConverter() { super(ElasticSearchConnection.class); @@ -38,6 +44,12 @@ public Object convert(Object object) { tryToConvert(elasticSearchConnection.getAuthType(), CONFIG_SOURCE_CLASSES) .ifPresent(elasticSearchConnection::setAuthType); + // `SSLConfig.certificates` is a oneOf of its own, so it needs a second converter pass. + if (elasticSearchConnection.getSslConfig() != null) { + convertProperty( + elasticSearchConnection.getSslConfig(), CERTIFICATES, SSL_CERTIFICATE_CLASSES); + } + return elasticSearchConnection; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/HiveConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/HiveConnectionClassConverter.java index fc3e0acf0d2e..98022fdaeeaf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/HiveConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/HiveConnectionClassConverter.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; import org.openmetadata.schema.services.connections.database.HiveConnection; import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.PostgresConnection; @@ -26,6 +27,8 @@ public class HiveConnectionClassConverter extends ClassConverter { private static final List> CONFIG_SOURCE_CLASSES = List.of(MysqlConnection.class, PostgresConnection.class); + private static final List> SSL_SOURCE_CLASSES = List.of(ValidateSSLClientConfig.class); + public HiveConnectionClassConverter() { super(HiveConnection.class); } @@ -39,6 +42,8 @@ public Object convert(Object object) { .ifPresent(hiveConnection::setMetastoreConnection); } + convertProperty(hiveConnection, "sslConfig", SSL_SOURCE_CLASSES); + return hiveConnection; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NestedConfigClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NestedConfigClassConverter.java new file mode 100644 index 000000000000..185271d0eaa1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NestedConfigClassConverter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import java.util.Map; +import org.openmetadata.schema.utils.JsonUtils; + +/** + * Converter for connections whose only need is to re-type their {@code Object} properties -- the + * ones a JSON Schema {@code oneOf} produces -- into the concrete classes that {@code oneOf} allows. + * + *

Prefer this over a bespoke converter class when the mapping is nothing but property name to + * candidate classes; write a dedicated {@link ClassConverter} when the connection needs more. + */ +public class NestedConfigClassConverter extends ClassConverter { + + private final Map>> objectProperties; + + public NestedConfigClassConverter(Class clazz, Map>> objectProperties) { + super(clazz); + this.objectProperties = Map.copyOf(objectProperties); + } + + @Override + public Object convert(Object object) { + Object connection = JsonUtils.convertValue(object, this.clazz); + objectProperties.forEach( + (property, candidates) -> convertProperty(connection, property, candidates)); + return connection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/OpenSearchConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/OpenSearchConnectionClassConverter.java new file mode 100644 index 000000000000..e902515e7ce8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/OpenSearchConnectionClassConverter.java @@ -0,0 +1,50 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import static org.openmetadata.service.secrets.converter.ElasticSearchConnectionClassConverter.CERTIFICATES; +import static org.openmetadata.service.secrets.converter.ElasticSearchConnectionClassConverter.SSL_CERTIFICATE_CLASSES; + +import java.util.List; +import org.openmetadata.schema.security.credentials.AWSCredentials; +import org.openmetadata.schema.services.connections.search.OpenSearchConnection; +import org.openmetadata.schema.services.connections.search.elasticSearch.ESBasicAuth; +import org.openmetadata.schema.utils.JsonUtils; + +/** Converter class to get an `OpenSearchConnection` object. */ +public class OpenSearchConnectionClassConverter extends ClassConverter { + + private static final List> CONFIG_SOURCE_CLASSES = + List.of(ESBasicAuth.class, AWSCredentials.class); + + public OpenSearchConnectionClassConverter() { + super(OpenSearchConnection.class); + } + + @Override + public Object convert(Object object) { + OpenSearchConnection openSearchConnection = + (OpenSearchConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvert(openSearchConnection.getAuthType(), CONFIG_SOURCE_CLASSES) + .ifPresent(openSearchConnection::setAuthType); + + // `SSLConfig.certificates` is a oneOf of its own, so it needs a second converter pass. + if (openSearchConnection.getSslConfig() != null) { + convertProperty(openSearchConnection.getSslConfig(), CERTIFICATES, SSL_CERTIFICATE_CLASSES); + } + + return openSearchConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/PrefectConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/PrefectConnectionClassConverter.java index c5e2eb91c801..f6c17778095b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/PrefectConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/PrefectConnectionClassConverter.java @@ -1,6 +1,7 @@ package org.openmetadata.service.secrets.converter; import java.util.List; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; import org.openmetadata.schema.services.connections.pipeline.PrefectConnection; import org.openmetadata.schema.services.connections.pipeline.prefect.CloudAuth; import org.openmetadata.schema.services.connections.pipeline.prefect.ServerAuth; @@ -14,6 +15,8 @@ public class PrefectConnectionClassConverter extends ClassConverter { private static final List> AUTH_TYPE_CLASSES = List.of(CloudAuth.class, ServerAuth.class); + private static final List> SSL_SOURCE_CLASSES = List.of(ValidateSSLClientConfig.class); + public PrefectConnectionClassConverter() { super(PrefectConnection.class); } @@ -26,6 +29,8 @@ public Object convert(Object object) { tryToConvert(prefectConnection.getAuthType(), AUTH_TYPE_CLASSES) .ifPresent(prefectConnection::setAuthType); + convertProperty(prefectConnection, "sslConfig", SSL_SOURCE_CLASSES); + return prefectConnection; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TableauConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TableauConnectionClassConverter.java index 79555d0de0d1..47d92022f72d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TableauConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TableauConnectionClassConverter.java @@ -3,6 +3,7 @@ import java.util.List; import org.openmetadata.schema.security.credentials.AccessTokenAuth; import org.openmetadata.schema.security.credentials.BasicAuth; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; import org.openmetadata.schema.services.connections.dashboard.TableauConnection; import org.openmetadata.schema.utils.JsonUtils; @@ -10,6 +11,8 @@ public class TableauConnectionClassConverter extends ClassConverter { private static final List> CONNECTION_CLASSES = List.of(BasicAuth.class, AccessTokenAuth.class); + private static final List> SSL_SOURCE_CLASSES = List.of(ValidateSSLClientConfig.class); + public TableauConnectionClassConverter() { super(TableauConnection.class); } @@ -22,6 +25,8 @@ public Object convert(Object object) { tryToConvertOrFail(tableauConnection.getAuthType(), CONNECTION_CLASSES) .ifPresent(tableauConnection::setAuthType); + convertProperty(tableauConnection, "sslConfig", SSL_SOURCE_CLASSES); + return tableauConnection; } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/DBSecretsManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/DBSecretsManagerTest.java index f74f521b61d6..97c47c4dd84c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/DBSecretsManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/DBSecretsManagerTest.java @@ -31,6 +31,8 @@ import org.openmetadata.schema.security.secrets.SecretsManagerProvider; import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.common.basicAuth; +import org.openmetadata.schema.services.connections.drive.SftpConnection; +import org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth; import org.openmetadata.schema.services.connections.mlmodel.SklearnConnection; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.fernet.Fernet; @@ -72,6 +74,26 @@ void testDecryptDatabaseServiceConnectionConfig() { testDecryptServiceConnection(); } + /** + * A secret behind a {@code oneOf} used to reach the database in the clear: the property is + * generated as a bare {@code Object}, so the encryption walk skipped the map Jackson left there. + */ + @Test + void testEncryptSecretNestedInAOneOfProperty() { + SftpConnection connection = + new SftpConnection() + .withHost("sftp.example.com") + .withAuthType( + new SftpBasicAuth().withUsername("sftp-user").withPassword(DECRYPTED_VALUE)); + + SftpConnection encrypted = + (SftpConnection) + secretsManager.encryptServiceConnectionConfig( + connection, "Sftp", "test", ServiceType.DRIVE); + + assertEquals(ENCRYPTED_VALUE, ((SftpBasicAuth) encrypted.getAuthType()).getPassword()); + } + @Test void testEncryptServiceConnectionWithoutPassword() { SklearnConnection connection = new SklearnConnection(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/NestedSecretConverterCoverageTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/NestedSecretConverterCoverageTest.java new file mode 100644 index 000000000000..82d0ab515b15 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/NestedSecretConverterCoverageTest.java @@ -0,0 +1,294 @@ +package org.openmetadata.service.secrets.converter; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.github.classgraph.ClassGraph; +import io.github.classgraph.Resource; +import io.github.classgraph.ScanResult; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.utils.JsonUtils; + +/** + * Secrets that live behind a JSON-Schema {@code oneOf} are generated as a bare {@code Object} field + * (jsonschema2pojo ignores {@code oneOf}), so at runtime they hold a {@code LinkedHashMap}. Both + * {@code PasswordEntityMasker} and {@code SecretsManager} only recurse into values whose package is + * {@code org.openmetadata.*}, so such a map is skipped entirely: its {@code format: password} + * leaves are never Fernet-encrypted, never handed to the secrets manager, and never masked on read. + * + *

The {@link ClassConverter} registered for the connection is what turns that map back into a + * typed object. This test walks every connection schema on the classpath and fails when a + * secret-bearing {@code oneOf} property has no converter that converts it, so a new connector + * cannot reintroduce the leak. + */ +class NestedSecretConverterCoverageTest { + + private static final String SECRET = "openmetadata-nested-secret"; + private static final String SCHEMA_ROOT = "json/schema/"; + private static final String CONNECTIONS_PATH = SCHEMA_ROOT + "entity/services/connections"; + + /** + * Secret-bearing {@code oneOf} branches that jsonschema2pojo never materialised into a class, so + * no converter can be written until the branch is extracted into its own schema file. Shrink this + * set; never grow it. + */ + private static final Set KNOWN_UNCONVERTIBLE = + Set.of( + "MetastoreConfig.connection", + "NatsConnection.authType", + "QlikSenseConnection.certificates", + "SapS4HanaConnection.authType"); + + @Test + void everySecretBearingOneOfPropertyHasAConverter() { + Map schemas = loadSchemas(); + assertTrue( + schemas.size() > 100, "No JSON schemas found on the classpath, found " + schemas.size()); + + List uncovered = new ArrayList<>(); + List staleAllowlist = new ArrayList<>(KNOWN_UNCONVERTIBLE); + + for (Map.Entry entry : schemas.entrySet()) { + String path = entry.getKey(); + // serviceConnection.json is the union of every connection config; covered via its members. + if (!path.startsWith(CONNECTIONS_PATH) || path.endsWith("serviceConnection.json")) { + continue; + } + JsonNode schema = entry.getValue(); + String connection = simpleJavaName(schema, path); + JsonNode properties = schema.path("properties"); + for (String property : iterable(properties.fieldNames())) { + Ref oneOf = resolveToOneOfNode(properties.get(property), path, schemas); + if (oneOf == null || !containsPasswordFormat(oneOf.node(), oneOf.path(), schemas)) { + continue; + } + String id = connection + "." + property; + if (staleAllowlist.remove(id)) { + continue; + } + if (leavesSecretsInAMap(schema, property, oneOf, schemas)) { + uncovered.add(id); + } + } + } + + assertTrue( + uncovered.isEmpty(), + "These connection properties hold secrets behind a oneOf but no ClassConverter converts " + + "them, so the secrets are stored unencrypted and returned unmasked: " + + uncovered); + assertTrue( + staleAllowlist.isEmpty(), + "KNOWN_UNCONVERTIBLE lists properties that are no longer secret-bearing oneOf properties; " + + "remove them: " + + staleAllowlist); + } + + /** + * Runs the registered converter over a payload built from the secret-bearing branch and reports + * whether the property is still a {@link Map} afterwards. A map is exactly what the password + * walkers skip, so a map here means the secrets below it stay in the clear. + */ + private boolean leavesSecretsInAMap( + JsonNode schema, String property, Ref oneOf, Map schemas) { + Class connectionClass; + try { + connectionClass = Class.forName(schema.path("javaType").asText("")); + } catch (ClassNotFoundException e) { + return false; // Not a generated class, so nothing walks it either. + } + Ref branch = secretBearingBranch(oneOf.node(), oneOf.path(), schemas); + Map payload = new LinkedHashMap<>(); + payload.put(property, secretPayload(branch.node(), branch.path(), schemas)); + try { + Object converted = ClassConverterFactory.getConverter(connectionClass).convert(payload); + String suffix = Character.toUpperCase(property.charAt(0)) + property.substring(1); + return connectionClass.getMethod("get" + suffix).invoke(converted) instanceof Map; + } catch (ReflectiveOperationException | RuntimeException e) { + return true; // Cannot be converted at all, so it is certainly not typed. + } + } + + private Ref secretBearingBranch(JsonNode oneOf, String path, Map schemas) { + JsonNode branches = oneOf.has("oneOf") ? oneOf.get("oneOf") : oneOf.get("anyOf"); + for (JsonNode branch : branches) { + Ref resolved = new Ref(branch, path); + String ref = branch.path("$ref").asText(""); + if (!ref.isEmpty()) { + resolved = resolveRef(ref, path, schemas); + } + if (resolved != null && containsPasswordFormat(resolved.node(), resolved.path(), schemas)) { + return resolved; + } + } + throw new IllegalStateException("No secret-bearing branch under " + path); + } + + /** + * The smallest payload that reaches every {@code format: password} leaf of {@code branch}, with + * each level's single-valued enum carried along so a converter candidate list can discriminate. + */ + private Object secretPayload(JsonNode branch, String path, Map schemas) { + Map payload = new LinkedHashMap<>(); + JsonNode properties = branch.path("properties"); + for (String name : iterable(properties.fieldNames())) { + JsonNode property = properties.get(name); + String ref = property.path("$ref").asText(""); + Ref resolved = ref.isEmpty() ? new Ref(property, path) : resolveRef(ref, path, schemas); + if (resolved == null) { + continue; + } + JsonNode node = resolved.node(); + if ("password".equals(node.path("format").asText(""))) { + payload.put(name, SECRET); + } else if (node.path("enum").size() == 1) { + payload.put(name, node.get("enum").get(0).asText()); + } else if (containsPasswordFormat(node, resolved.path(), schemas)) { + payload.put( + name, + node.has("properties") + ? secretPayload(node, resolved.path(), schemas) + : secretPayload( + secretBearingBranch(node, resolved.path(), schemas).node(), + resolved.path(), + schemas)); + } + } + return payload; + } + + private String simpleJavaName(JsonNode schema, String path) { + String javaType = schema.path("javaType").asText(""); + if (!javaType.isEmpty()) { + return javaType.substring(javaType.lastIndexOf('.') + 1); + } + String file = path.substring(path.lastIndexOf('/') + 1); + return file.substring(0, file.length() - ".json".length()); + } + + /** + * Returns the {@code oneOf}/{@code anyOf} node a property resolves to, or {@code null} when the + * property is not generated as a bare {@code Object} because it declares its own properties. + */ + private Ref resolveToOneOfNode(JsonNode property, String path, Map schemas) { + JsonNode node = property; + String base = path; + for (int depth = 0; depth < 8 && node != null; depth++) { + if (node.has("properties")) { + return null; + } + if (node.has("oneOf") || node.has("anyOf")) { + return new Ref(node, base); + } + String ref = node.path("$ref").asText(""); + if (ref.isEmpty()) { + return null; + } + Ref resolved = resolveRef(ref, base, schemas); + if (resolved == null) { + return null; + } + node = resolved.node(); + base = resolved.path(); + } + return null; + } + + private boolean containsPasswordFormat( + JsonNode node, String path, Map schemas) { + Deque queue = new ArrayDeque<>(); + Set visited = new HashSet<>(); + queue.add(new Ref(node, path)); + while (!queue.isEmpty()) { + Ref current = queue.poll(); + JsonNode value = current.node(); + if (value == null || !value.isObject()) { + continue; + } + if ("password".equals(value.path("format").asText(""))) { + return true; + } + String ref = value.path("$ref").asText(""); + if (!ref.isEmpty()) { + if (visited.add(current.path() + "|" + ref)) { + Ref resolved = resolveRef(ref, current.path(), schemas); + if (resolved != null) { + queue.add(resolved); + } + } + continue; + } + for (String keyword : List.of("oneOf", "anyOf", "allOf")) { + value.path(keyword).forEach(branch -> queue.add(new Ref(branch, current.path()))); + } + value.path("properties").forEach(child -> queue.add(new Ref(child, current.path()))); + if (value.path("items").isObject()) { + queue.add(new Ref(value.get("items"), current.path())); + } + } + return false; + } + + private Ref resolveRef(String ref, String base, Map schemas) { + int hash = ref.indexOf('#'); + String filePart = hash >= 0 ? ref.substring(0, hash) : ref; + String fragment = hash >= 0 ? ref.substring(hash + 1) : ""; + String targetPath = filePart.isEmpty() ? base : normalize(base, filePart); + JsonNode node = schemas.get(targetPath); + if (node == null) { + return null; + } + for (String segment : fragment.split("/")) { + if (!segment.isEmpty()) { + node = node.path(segment); + } + } + return node.isMissingNode() ? null : new Ref(node, targetPath); + } + + /** Resolves a schema-relative {@code $ref} file path against the referring schema's path. */ + private String normalize(String base, String relative) { + Deque segments = new ArrayDeque<>(); + for (String segment : base.substring(0, base.lastIndexOf('/')).split("/")) { + segments.addLast(segment); + } + for (String segment : relative.split("/")) { + if (segment.equals("..")) { + segments.pollLast(); + } else if (!segment.equals(".") && !segment.isEmpty()) { + segments.addLast(segment); + } + } + return String.join("/", segments); + } + + /** Every schema under {@code json/schema/}, keyed by its classpath-relative path. */ + private Map loadSchemas() { + Map schemas = new TreeMap<>(); + try (ScanResult scan = new ClassGraph().acceptPaths(SCHEMA_ROOT).scan()) { + for (Resource resource : scan.getResourcesWithExtension("json")) { + try { + schemas.put(resource.getPath(), JsonUtils.readTree(resource.getContentAsString())); + } catch (Exception ignored) { + // Not every JSON under json/schema parses as a schema; those cannot declare secrets. + } + } + } + return schemas; + } + + private static Iterable iterable(java.util.Iterator iterator) { + return () -> iterator; + } + + private record Ref(JsonNode node, String path) {} +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java index a44fb7532411..71df0dc35a48 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java @@ -11,10 +11,14 @@ import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.cassandra.CloudConfig; import org.openmetadata.schema.services.connections.database.cassandra.CloudConfig__1; +import org.openmetadata.schema.services.connections.drive.SftpConnection; +import org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth; +import org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth; import org.openmetadata.service.exception.EntityMaskException; public class PasswordEntityMaskerTest extends TestEntityMasker { private static final String TOKEN = "openmetadata-token"; + private static final String SFTP_PASSWORD = "openmetadata-sftp-secret"; public PasswordEntityMaskerTest() { CONFIG.setMaskPasswordsAPI(true); @@ -66,6 +70,54 @@ private String astraToken(CassandraConnection connection) { return ((CloudConfig) connection.getAuthType()).getCloudConfig().getToken(); } + @Test + void testSftpPasswordIsMaskedAndRestored() { + SftpConnection original = + new SftpConnection() + .withHost("sftp.example.com") + .withAuthType( + new SftpBasicAuth().withUsername("sftp-user").withPassword(SFTP_PASSWORD)); + + SftpConnection masked = + (SftpConnection) + EntityMaskerFactory.createEntityMasker() + .maskServiceConnectionConfig(original, "Sftp", ServiceType.DRIVE); + assertEquals(getMaskedPassword(), sftpPassword(masked)); + assertEquals("sftp-user", sftpAuth(masked).getUsername()); + + SftpConnection restored = + (SftpConnection) + EntityMaskerFactory.createEntityMasker() + .unmaskServiceConnectionConfig(masked, original, "Sftp", ServiceType.DRIVE); + assertEquals(SFTP_PASSWORD, sftpPassword(restored)); + } + + /** The config arrives from the database as a map, which is what used to defeat the masker. */ + @Test + void testSftpPrivateKeyFromSerializedConfigIsMasked() { + Map serialized = + Map.of( + "host", + "sftp.example.com", + "authType", + Map.of("username", "sftp-user", "privateKey", SFTP_PASSWORD)); + + SftpConnection masked = + (SftpConnection) + EntityMaskerFactory.createEntityMasker() + .maskServiceConnectionConfig(serialized, "Sftp", ServiceType.DRIVE); + + assertEquals(getMaskedPassword(), ((SftpKeyAuth) masked.getAuthType()).getPrivateKey()); + } + + private SftpBasicAuth sftpAuth(SftpConnection connection) { + return (SftpBasicAuth) connection.getAuthType(); + } + + private String sftpPassword(SftpConnection connection) { + return sftpAuth(connection).getPassword(); + } + @Test void testExceptionConnection() { Map mysqlConnectionObject = diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/common/sslConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/common/sslConfig.json index ccdabd29c793..6abe34fd3396 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/common/sslConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/common/sslConfig.json @@ -8,7 +8,6 @@ "type": "object", "properties": { "certificates": { - "type":"object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/basicAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/basicAuth.json new file mode 100644 index 000000000000..21e49df8c4b0 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/basicAuth.json @@ -0,0 +1,23 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/drive/sftp/basicAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Username/Password Authentication", + "description": "Username and password authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "password": { + "title": "Password", + "description": "SFTP password", + "type": "string", + "format": "password" + } + }, + "required": ["username", "password"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/keyAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/keyAuth.json new file mode 100644 index 000000000000..4d11b88f9701 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftp/keyAuth.json @@ -0,0 +1,30 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/drive/sftp/keyAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Private Key Authentication", + "description": "SSH private key authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "privateKey": { + "title": "Private Key", + "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", + "type": "string", + "format": "password", + "uiFieldType": "fileOrInput" + }, + "privateKeyPassphrase": { + "title": "Private Key Passphrase", + "description": "Passphrase for the private key (if encrypted)", + "type": "string", + "format": "password" + } + }, + "required": ["username", "privateKey"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftpConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftpConnection.json index 81b1bb99f53e..1acbb93a80d6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftpConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/drive/sftpConnection.json @@ -11,53 +11,6 @@ "type": "string", "enum": ["Sftp"], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": ["username", "password"], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": ["username", "privateKey"], - "additionalProperties": false } }, "properties": { @@ -83,10 +36,10 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { - "$ref": "#/definitions/basicAuth" + "$ref": "sftp/basicAuth.json" }, { - "$ref": "#/definitions/keyAuth" + "$ref": "sftp/keyAuth.json" } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/common/sslConfig.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/common/sslConfig.json index a34f422ce6ee..773615a33d50 100644 --- a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/common/sslConfig.json +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/common/sslConfig.json @@ -6,7 +6,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/basicAuth.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/basicAuth.json new file mode 100644 index 000000000000..d107b21bea70 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/basicAuth.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Username/Password Authentication", + "description": "Username and password authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "password": { + "title": "Password", + "description": "SFTP password", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "password" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/keyAuth.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/keyAuth.json new file mode 100644 index 000000000000..5af01d3054b5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftp/keyAuth.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Private Key Authentication", + "description": "SSH private key authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "privateKey": { + "title": "Private Key", + "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", + "type": "string", + "format": "password", + "uiFieldType": "fileOrInput" + }, + "privateKeyPassphrase": { + "title": "Private Key Passphrase", + "description": "Passphrase for the private key (if encrypted)", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "privateKey" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftpConnection.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftpConnection.json index ea4749b74a69..b869eca54fc5 100644 --- a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftpConnection.json +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/drive/sftpConnection.json @@ -12,59 +12,6 @@ "Sftp" ], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "password" - ], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "privateKey" - ], - "additionalProperties": false } }, "properties": { @@ -93,9 +40,11 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Username/Password Authentication", "description": "Username and password authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", "properties": { "username": { "title": "Username", @@ -116,9 +65,11 @@ "additionalProperties": false }, { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Private Key Authentication", "description": "SSH private key authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", "properties": { "username": { "title": "Username", diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/elasticSearchConnection.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/elasticSearchConnection.json index 0379b9ed9857..f07ddacdb105 100644 --- a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/elasticSearchConnection.json +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/elasticSearchConnection.json @@ -86,7 +86,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/openSearchConnection.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/openSearchConnection.json index 94f9022a68b8..3bd6040aaa33 100644 --- a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/openSearchConnection.json +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/search/openSearchConnection.json @@ -144,7 +144,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ diff --git a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/serviceConnection.json b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/serviceConnection.json index 201b8b7a8192..122032e2955b 100644 --- a/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/serviceConnection.json +++ b/openmetadata-ui/src/main/resources/ui/public/jsons/connectionSchemas/connections/serviceConnection.json @@ -41143,7 +41143,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -41405,7 +41404,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -42412,59 +42410,6 @@ "Sftp" ], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "password" - ], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "privateKey" - ], - "additionalProperties": false } }, "properties": { @@ -42493,9 +42438,11 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Username/Password Authentication", "description": "Username and password authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", "properties": { "username": { "title": "Username", @@ -42516,9 +42463,11 @@ "additionalProperties": false }, { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Private Key Authentication", "description": "SSH private key authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", "properties": { "username": { "title": "Username", @@ -84267,7 +84216,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -84529,7 +84477,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -85536,59 +85483,6 @@ "Sftp" ], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "password" - ], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "privateKey" - ], - "additionalProperties": false } }, "properties": { @@ -85617,9 +85511,11 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Username/Password Authentication", "description": "Username and password authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", "properties": { "username": { "title": "Username", @@ -85640,9 +85536,11 @@ "additionalProperties": false }, { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "Private Key Authentication", "description": "SSH private key authentication for SFTP", "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", "properties": { "username": { "title": "Username", diff --git a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json index 42cd8015fa61..029bb36d2fc0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json +++ b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json @@ -15093,7 +15093,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -15665,59 +15664,6 @@ "Sftp" ], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "password" - ], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "privateKey" - ], - "additionalProperties": false } }, "properties": { @@ -15743,10 +15689,61 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { - "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/10/properties/config/oneOf/2/definitions/basicAuth" + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Username/Password Authentication", + "description": "Username and password authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "password": { + "title": "Password", + "description": "SFTP password", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "password" + ], + "additionalProperties": false }, { - "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/10/properties/config/oneOf/2/definitions/keyAuth" + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Private Key Authentication", + "description": "SSH private key authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "privateKey": { + "title": "Private Key", + "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", + "type": "string", + "format": "password", + "uiFieldType": "fileOrInput" + }, + "privateKeyPassphrase": { + "title": "Private Key Passphrase", + "description": "Passphrase for the private key (if encrypted)", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "privateKey" + ], + "additionalProperties": false } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json index 8341493b87cb..4fc673f3a452 100644 --- a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json +++ b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json @@ -20034,7 +20034,6 @@ "type": "object", "properties": { "certificates": { - "type": "object", "title": "SSL Certificates", "description": "SSL Certificates", "oneOf": [ @@ -20606,59 +20605,6 @@ "Sftp" ], "default": "Sftp" - }, - "basicAuth": { - "title": "Username/Password Authentication", - "description": "Username and password authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "password": { - "title": "Password", - "description": "SFTP password", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "password" - ], - "additionalProperties": false - }, - "keyAuth": { - "title": "Private Key Authentication", - "description": "SSH private key authentication for SFTP", - "type": "object", - "properties": { - "username": { - "title": "Username", - "description": "SFTP username", - "type": "string" - }, - "privateKey": { - "title": "Private Key", - "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", - "type": "string", - "format": "password", - "uiFieldType": "fileOrInput" - }, - "privateKeyPassphrase": { - "title": "Private Key Passphrase", - "description": "Passphrase for the private key (if encrypted)", - "type": "string", - "format": "password" - } - }, - "required": [ - "username", - "privateKey" - ], - "additionalProperties": false } }, "properties": { @@ -20684,10 +20630,61 @@ "description": "Authentication method: username/password or SSH private key", "oneOf": [ { - "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/10/properties/config/oneOf/2/definitions/basicAuth" + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Username/Password Authentication", + "description": "Username and password authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpBasicAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "password": { + "title": "Password", + "description": "SFTP password", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "password" + ], + "additionalProperties": false }, { - "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/10/properties/config/oneOf/2/definitions/keyAuth" + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Private Key Authentication", + "description": "SSH private key authentication for SFTP", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.drive.sftp.SftpKeyAuth", + "properties": { + "username": { + "title": "Username", + "description": "SFTP username", + "type": "string" + }, + "privateKey": { + "title": "Private Key", + "description": "SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.", + "type": "string", + "format": "password", + "uiFieldType": "fileOrInput" + }, + "privateKeyPassphrase": { + "title": "Private Key Passphrase", + "description": "Passphrase for the private key (if encrypted)", + "type": "string", + "format": "password" + } + }, + "required": [ + "username", + "privateKey" + ], + "additionalProperties": false } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.test.tsx new file mode 100644 index 000000000000..89e579a75342 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.test.tsx @@ -0,0 +1,157 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render, screen } from '@testing-library/react'; +import { + getKeyValues, + getSchemaProperties, +} from './ServiceConnectionDetailsUtils'; + +const SFTP_SCHEMA = { + properties: { + host: { title: 'Host', type: 'string' }, + authType: { + title: 'Authentication Type', + oneOf: [ + { + title: 'Username/Password Authentication', + type: 'object', + properties: { + username: { title: 'Username', type: 'string' }, + password: { title: 'Password', type: 'string', format: 'password' }, + }, + }, + { + title: 'Private Key Authentication', + type: 'object', + properties: { + username: { title: 'Username', type: 'string' }, + privateKey: { + title: 'Private Key', + type: 'string', + format: 'password', + }, + }, + }, + ], + }, + }, +}; + +describe('getSchemaProperties', () => { + it('resolves the properties of the oneOf branch matching the stored value', () => { + expect( + getSchemaProperties( + SFTP_SCHEMA.properties.authType, + { username: 'sftp-user', password: '*********' }, + SFTP_SCHEMA + ) + ).toEqual({ + username: { title: 'Username', type: 'string' }, + password: { title: 'Password', type: 'string', format: 'password' }, + privateKey: { title: 'Private Key', type: 'string', format: 'password' }, + }); + }); + + it('keeps every branch so a secret declared in any of them stays marked', () => { + const resolved = getSchemaProperties( + SFTP_SCHEMA.properties.authType, + {}, + SFTP_SCHEMA + ); + + expect(resolved.password).toHaveProperty('format', 'password'); + expect(resolved.privateKey).toHaveProperty('format', 'password'); + }); + + it('follows a local $ref before reading the branches', () => { + const schema = { + definitions: { auth: SFTP_SCHEMA.properties.authType }, + properties: { authType: { $ref: '#/definitions/auth' } }, + }; + + expect( + getSchemaProperties(schema.properties.authType, { password: 'x' }, schema) + ).toHaveProperty('password.format', 'password'); + }); + + it('returns an empty object for a property with neither properties nor branches', () => { + expect(getSchemaProperties({ type: 'string' }, {}, SFTP_SCHEMA)).toEqual( + {} + ); + expect(getSchemaProperties(undefined, {}, SFTP_SCHEMA)).toEqual({}); + }); + + it('stops at a $ref that points nowhere instead of throwing', () => { + const schema = { + definitions: {}, + properties: { authType: { $ref: '#/definitions/missing' } }, + }; + + expect(getSchemaProperties(schema.properties.authType, {}, schema)).toEqual( + {} + ); + }); + + it('does not loop on a self-referential $ref', () => { + const schema = { + definitions: { loop: { $ref: '#/definitions/loop' } }, + properties: { authType: { $ref: '#/definitions/loop' } }, + }; + + expect(getSchemaProperties(schema.properties.authType, {}, schema)).toEqual( + {} + ); + }); +}); + +describe('getKeyValues', () => { + const renderConnection = (connection: Record) => + render( +

+ {getKeyValues({ + obj: connection, + schemaPropertyObject: SFTP_SCHEMA.properties, + schema: SFTP_SCHEMA, + serviceCategory: 'driveServices', + })} +
+ ); + + it('renders a secret nested in a oneOf branch as a password input', () => { + renderConnection({ + host: 'sftp.example.com', + authType: { username: 'sftp-user', password: '*********' }, + }); + + const inputs = screen.getAllByTestId('input-field'); + const password = inputs.find( + (input) => (input as HTMLInputElement).value === '*********' + ); + + expect(password).toHaveAttribute('type', 'password'); + }); + + it('leaves non-secret fields readable', () => { + renderConnection({ + host: 'sftp.example.com', + authType: { username: 'sftp-user', password: '*********' }, + }); + + const username = screen + .getAllByTestId('input-field') + .find((input) => (input as HTMLInputElement).value === 'sftp-user'); + + expect(username).toHaveAttribute('type', 'text'); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.tsx index 6c7ba74b8baa..fdabe7025f1a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceConnectionDetailsUtils.tsx @@ -13,7 +13,15 @@ import { InfoCircleOutlined } from '@ant-design/icons'; import { Col, Input, Row, Select, Space, Tooltip, Typography } from 'antd'; -import { get, isArray, isEmpty, isNull, isObject, startCase } from 'lodash'; +import { + get, + isArray, + isEmpty, + isNull, + isObject, + isString, + startCase, +} from 'lodash'; import { ReactNode } from 'react'; import ErrorPlaceHolder from '../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { FILTER_PATTERN_BY_SERVICE_TYPE } from '../constants/ServiceConnection.constants'; @@ -126,6 +134,76 @@ const renderFilterPattern = ( ); }; +const MAX_SCHEMA_RESOLUTION_DEPTH = 10; + +// Follows a local `#/definitions/...` pointer, as deep as the schema chains them. +const resolveRef = ( + node: Record, + schema: Record +): Record => { + let current = node; + for (let depth = 0; depth < MAX_SCHEMA_RESOLUTION_DEPTH; depth++) { + const ref = current?.$ref; + if (!isString(ref) || !ref.startsWith('#/')) { + return current; + } + const resolved = get(schema, ref.slice(2).split('/')); + if (!isObject(resolved)) { + return current; + } + current = resolved as Record; + } + + return current; +}; + +/** + * Resolves the `properties` of a nested config, walking `$ref` and `oneOf`/`anyOf`. + * + * Many connectors keep their credentials inside a `oneOf` branch -- SFTP `authType`, every + * `sslConfig`, the Alation/Databricks/OpenSearch auth types. Reading `schemaProperty.properties` + * alone yields `{}` for those, which drops the `format: password` marker and renders the secret + * as a readable text input. Branches are merged so a secret declared in any branch stays masked, + * with the branch matching the stored value last so it wins on the fields it shares. + */ +export const getSchemaProperties = ( + schemaProperty: unknown, + value: unknown, + schema: Record, + depth = 0 +): Record => { + if (!isObject(schemaProperty) || depth >= MAX_SCHEMA_RESOLUTION_DEPTH) { + return {}; + } + + const node = resolveRef(schemaProperty as Record, schema); + if (isObject(node.properties)) { + return node.properties as Record; + } + + const branches = node.oneOf ?? node.anyOf; + if (!isArray(branches)) { + return {}; + } + + const valueKeys = isObject(value) ? Object.keys(value) : []; + let merged: Record = {}; + let bestMatch: Record = {}; + let bestScore = 0; + + branches.forEach((branch) => { + const properties = getSchemaProperties(branch, value, schema, depth + 1); + merged = { ...properties, ...merged }; + const score = valueKeys.filter((key) => key in properties).length; + if (score > bestScore) { + bestScore = score; + bestMatch = properties; + } + }); + + return { ...merged, ...bestMatch }; +}; + export const getKeyValues = ({ obj, schemaPropertyObject, @@ -234,18 +312,6 @@ const handleSpecialServiceConfig = ( }); } - // Database service - GCP credentials - if (serviceType === EntityType.DATABASE_SERVICE && key === 'credentials') { - const gcpSchema = schemaPropertyObject[key].definitions.gcpCredentialsPath; - - return getKeyValues({ - obj: value, - schemaPropertyObject: gcpSchema, - schema, - serviceCategory, - }); - } - // Metadata service - Security config if (serviceType === EntityType.METADATA_SERVICE && key === 'securityConfig') { return renderOneOfSchema({ @@ -293,10 +359,13 @@ const handleDatabaseConfigSource = ( 'definitions.GCPConfig.properties.securityConfig.definitions.GCPValues.properties', {} ) - : get( - schema, - 'definitions.GCPConfig.properties.securityConfig.definitions.gcpCredentialsPath', - {} + : getSchemaProperties( + get( + schema, + 'definitions.GCPConfig.properties.securityConfig.definitions.gcpCredentialsPath' + ), + value, + schema ); return getKeyValues({ @@ -336,7 +405,11 @@ const handleDatabaseConfigSource = ( return getKeyValues({ obj: value, - schemaPropertyObject: schema.definitions[definition], + schemaPropertyObject: getSchemaProperties( + get(schema, ['definitions', String(definition)]), + value, + schema + ), schema, serviceCategory, }); @@ -388,7 +461,11 @@ const getNestedConfigValue = ({ return getKeyValues({ obj: value, - schemaPropertyObject: schemaPropertyObject[key]?.properties ?? {}, + schemaPropertyObject: getSchemaProperties( + schemaPropertyObject[key], + value, + schema + ), schema, serviceCategory, }); From 5dd77ad00cef6d35fdb828cc2a8fb31bf6e7e5c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 16 Sep 2026 08:45:51 +0000 Subject: [PATCH 2/2] Update generated TypeScript types and dereferenced schemas --- .../connections/drive/sftp/basicAuth.ts | 25 ++++++++++++++++ .../connections/drive/sftp/keyAuth.ts | 29 +++++++++++++++++++ .../ingestionSchemas/testSuitePipeline.json | 4 +-- .../src/jsons/ingestionSchemas/workflow.json | 4 +-- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/basicAuth.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/keyAuth.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/basicAuth.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/basicAuth.ts new file mode 100644 index 000000000000..8b7816100fcd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/basicAuth.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Username and password authentication for SFTP + */ +export interface BasicAuth { + /** + * SFTP password + */ + password: string; + /** + * SFTP username + */ + username: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/keyAuth.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/keyAuth.ts new file mode 100644 index 000000000000..cd4b1eca7215 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/drive/sftp/keyAuth.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * SSH private key authentication for SFTP + */ +export interface KeyAuth { + /** + * SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys. + */ + privateKey: string; + /** + * Passphrase for the private key (if encrypted) + */ + privateKeyPassphrase?: string; + /** + * SFTP username + */ + username: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json index 029bb36d2fc0..93516f541da5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json +++ b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/testSuitePipeline.json @@ -10212,7 +10212,7 @@ "type": { "title": "Service Type", "description": "Service Type", - "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/56/definitions/data360Type", + "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/57/definitions/data360Type", "default": "Data360" }, "consumerKey": { @@ -10278,7 +10278,7 @@ "type": { "title": "Service Type", "description": "Service Type", - "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/57/definitions/sapBw4HanaType", + "$ref": "#/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/58/definitions/sapBw4HanaType", "default": "SapBw4Hana" }, "hostPort": { diff --git a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json index 4fc673f3a452..b629aa5a4d28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json +++ b/openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas/workflow.json @@ -15709,7 +15709,7 @@ "type": { "title": "Service Type", "description": "Service Type", - "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/56/definitions/data360Type", + "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/57/definitions/data360Type", "default": "Data360" }, "consumerKey": { @@ -15775,7 +15775,7 @@ "type": { "title": "Service Type", "description": "Service Type", - "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/57/definitions/sapBw4HanaType", + "$ref": "#/definitions/source/properties/sourceConfig/properties/config/oneOf/14/definitions/serviceConnections/properties/serviceConnection/oneOf/2/properties/config/oneOf/58/definitions/sapBw4HanaType", "default": "SapBw4Hana" }, "hostPort": {