diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 80f0f7fa2..2eb1912ec 100755 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -3c4239f03d7d476d7c9b3d60bc668451bd573863 \ No newline at end of file +ea4ac909fc4e014967c50f979e955b8275b076e4 \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index b63ce582d..9eb378a54 100755 --- a/.gitattributes +++ b/.gitattributes @@ -2270,6 +2270,8 @@ databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Ingestion databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/IngestionSourceType.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/JiraConnectorOptions.java linguist-generated=true +databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/JsonTransformerOptions.java linguist-generated=true +databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/KafkaOptions.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ListPipelineEventsRequest.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ListPipelineEventsResponse.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ListPipelinesRequest.java linguist-generated=true @@ -2337,6 +2339,8 @@ databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TableSpec databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TikTokAdsOptions.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TikTokAdsOptionsTikTokDataLevel.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TikTokAdsOptionsTikTokReportType.java linguist-generated=true +databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Transformer.java linguist-generated=true +databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TransformerFormat.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Truncation.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TruncationTruncationDetail.java linguist-generated=true databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/UpdateInfo.java linguist-generated=true diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index a0e89431d..e5c1ab8d1 100755 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -44,3 +44,4 @@ * Add `accountApi`, `accountDatabricksOne` and `accountUi` fields for `com.databricks.sdk.service.settings.CustomerFacingIngressNetworkPolicyRequestDestination`. * Add `scopeQualifier` field for `com.databricks.sdk.service.settings.CustomerFacingIngressNetworkPolicyWorkspaceApiDestination`. * Add `displayName` and `previewPhase` fields for `com.databricks.sdk.service.settingsv2.SettingsMetadata`. +* Add `kafkaOptions` field for `com.databricks.sdk.service.pipelines.ConnectorOptions`. \ No newline at end of file diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ConnectorOptions.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ConnectorOptions.java index 37b0e3145..c8e531a91 100755 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ConnectorOptions.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/ConnectorOptions.java @@ -26,6 +26,10 @@ public class ConnectorOptions { @JsonProperty("jira_options") private JiraConnectorOptions jiraOptions; + /** */ + @JsonProperty("kafka_options") + private KafkaOptions kafkaOptions; + /** */ @JsonProperty("meta_ads_options") private MetaMarketingOptions metaAdsOptions; @@ -86,6 +90,15 @@ public JiraConnectorOptions getJiraOptions() { return jiraOptions; } + public ConnectorOptions setKafkaOptions(KafkaOptions kafkaOptions) { + this.kafkaOptions = kafkaOptions; + return this; + } + + public KafkaOptions getKafkaOptions() { + return kafkaOptions; + } + public ConnectorOptions setMetaAdsOptions(MetaMarketingOptions metaAdsOptions) { this.metaAdsOptions = metaAdsOptions; return this; @@ -149,6 +162,7 @@ public boolean equals(Object o) { && Objects.equals(gdriveOptions, that.gdriveOptions) && Objects.equals(googleAdsOptions, that.googleAdsOptions) && Objects.equals(jiraOptions, that.jiraOptions) + && Objects.equals(kafkaOptions, that.kafkaOptions) && Objects.equals(metaAdsOptions, that.metaAdsOptions) && Objects.equals(outlookOptions, that.outlookOptions) && Objects.equals(sharepointOptions, that.sharepointOptions) @@ -164,6 +178,7 @@ public int hashCode() { gdriveOptions, googleAdsOptions, jiraOptions, + kafkaOptions, metaAdsOptions, outlookOptions, sharepointOptions, @@ -179,6 +194,7 @@ public String toString() { .add("gdriveOptions", gdriveOptions) .add("googleAdsOptions", googleAdsOptions) .add("jiraOptions", jiraOptions) + .add("kafkaOptions", kafkaOptions) .add("metaAdsOptions", metaAdsOptions) .add("outlookOptions", outlookOptions) .add("sharepointOptions", sharepointOptions) diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/JsonTransformerOptions.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/JsonTransformerOptions.java new file mode 100755 index 000000000..abfec380b --- /dev/null +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/JsonTransformerOptions.java @@ -0,0 +1,105 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package com.databricks.sdk.service.pipelines; + +import com.databricks.sdk.support.Generated; +import com.databricks.sdk.support.ToStringer; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; + +@Generated +public class JsonTransformerOptions { + /** Parse the entire value as a single Variant column. */ + @JsonProperty("as_variant") + private Boolean asVariant; + + /** Inline schema string for JSON parsing (Spark DDL format). */ + @JsonProperty("schema") + private String schema; + + /** (Optional) Schema evolution mode for schema inference. */ + @JsonProperty("schema_evolution_mode") + private FileIngestionOptionsSchemaEvolutionMode schemaEvolutionMode; + + /** Path to a schema file (.ddl). */ + @JsonProperty("schema_file_path") + private String schemaFilePath; + + /** (Optional) Schema hints as a comma-separated string of "column_name type" pairs. */ + @JsonProperty("schema_hints") + private String schemaHints; + + public JsonTransformerOptions setAsVariant(Boolean asVariant) { + this.asVariant = asVariant; + return this; + } + + public Boolean getAsVariant() { + return asVariant; + } + + public JsonTransformerOptions setSchema(String schema) { + this.schema = schema; + return this; + } + + public String getSchema() { + return schema; + } + + public JsonTransformerOptions setSchemaEvolutionMode( + FileIngestionOptionsSchemaEvolutionMode schemaEvolutionMode) { + this.schemaEvolutionMode = schemaEvolutionMode; + return this; + } + + public FileIngestionOptionsSchemaEvolutionMode getSchemaEvolutionMode() { + return schemaEvolutionMode; + } + + public JsonTransformerOptions setSchemaFilePath(String schemaFilePath) { + this.schemaFilePath = schemaFilePath; + return this; + } + + public String getSchemaFilePath() { + return schemaFilePath; + } + + public JsonTransformerOptions setSchemaHints(String schemaHints) { + this.schemaHints = schemaHints; + return this; + } + + public String getSchemaHints() { + return schemaHints; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JsonTransformerOptions that = (JsonTransformerOptions) o; + return Objects.equals(asVariant, that.asVariant) + && Objects.equals(schema, that.schema) + && Objects.equals(schemaEvolutionMode, that.schemaEvolutionMode) + && Objects.equals(schemaFilePath, that.schemaFilePath) + && Objects.equals(schemaHints, that.schemaHints); + } + + @Override + public int hashCode() { + return Objects.hash(asVariant, schema, schemaEvolutionMode, schemaFilePath, schemaHints); + } + + @Override + public String toString() { + return new ToStringer(JsonTransformerOptions.class) + .add("asVariant", asVariant) + .add("schema", schema) + .add("schemaEvolutionMode", schemaEvolutionMode) + .add("schemaFilePath", schemaFilePath) + .add("schemaHints", schemaHints) + .toString(); + } +} diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/KafkaOptions.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/KafkaOptions.java new file mode 100755 index 000000000..bdec85955 --- /dev/null +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/KafkaOptions.java @@ -0,0 +1,154 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package com.databricks.sdk.service.pipelines; + +import com.databricks.sdk.support.Generated; +import com.databricks.sdk.support.ToStringer; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; + +@Generated +public class KafkaOptions { + /** + * Undocumented backdoor mechanism for overriding parameters to pass to the Kafka client. This is + * not supported and may break at any time. + */ + @JsonProperty("client_config") + private Map clientConfig; + + /** (Optional) Transformer for the message key. If not specified, the key is left as raw bytes. */ + @JsonProperty("key_transformer") + private Transformer keyTransformer; + + /** Internal option to control the maximum number of offsets to process per trigger. */ + @JsonProperty("max_offsets_per_trigger") + private Long maxOffsetsPerTrigger; + + /** + * (Optional) Where to begin reading when no checkpoint exists. Valid values: "latest" and + * "earliest". Defaults to "latest". + */ + @JsonProperty("starting_offset") + private String startingOffset; + + /** + * Java regex pattern to subscribe to matching topics. Only one of topics or topic_pattern must be + * specified. + */ + @JsonProperty("topic_pattern") + private String topicPattern; + + /** Topics to subscribe to. Only one of topics or topic_pattern must be specified. */ + @JsonProperty("topics") + private Collection topics; + + /** + * (Optional) Transformer for the message value. If not specified, the value is left as raw bytes. + */ + @JsonProperty("value_transformer") + private Transformer valueTransformer; + + public KafkaOptions setClientConfig(Map clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + public Map getClientConfig() { + return clientConfig; + } + + public KafkaOptions setKeyTransformer(Transformer keyTransformer) { + this.keyTransformer = keyTransformer; + return this; + } + + public Transformer getKeyTransformer() { + return keyTransformer; + } + + public KafkaOptions setMaxOffsetsPerTrigger(Long maxOffsetsPerTrigger) { + this.maxOffsetsPerTrigger = maxOffsetsPerTrigger; + return this; + } + + public Long getMaxOffsetsPerTrigger() { + return maxOffsetsPerTrigger; + } + + public KafkaOptions setStartingOffset(String startingOffset) { + this.startingOffset = startingOffset; + return this; + } + + public String getStartingOffset() { + return startingOffset; + } + + public KafkaOptions setTopicPattern(String topicPattern) { + this.topicPattern = topicPattern; + return this; + } + + public String getTopicPattern() { + return topicPattern; + } + + public KafkaOptions setTopics(Collection topics) { + this.topics = topics; + return this; + } + + public Collection getTopics() { + return topics; + } + + public KafkaOptions setValueTransformer(Transformer valueTransformer) { + this.valueTransformer = valueTransformer; + return this; + } + + public Transformer getValueTransformer() { + return valueTransformer; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KafkaOptions that = (KafkaOptions) o; + return Objects.equals(clientConfig, that.clientConfig) + && Objects.equals(keyTransformer, that.keyTransformer) + && Objects.equals(maxOffsetsPerTrigger, that.maxOffsetsPerTrigger) + && Objects.equals(startingOffset, that.startingOffset) + && Objects.equals(topicPattern, that.topicPattern) + && Objects.equals(topics, that.topics) + && Objects.equals(valueTransformer, that.valueTransformer); + } + + @Override + public int hashCode() { + return Objects.hash( + clientConfig, + keyTransformer, + maxOffsetsPerTrigger, + startingOffset, + topicPattern, + topics, + valueTransformer); + } + + @Override + public String toString() { + return new ToStringer(KafkaOptions.class) + .add("clientConfig", clientConfig) + .add("keyTransformer", keyTransformer) + .add("maxOffsetsPerTrigger", maxOffsetsPerTrigger) + .add("startingOffset", startingOffset) + .add("topicPattern", topicPattern) + .add("topics", topics) + .add("valueTransformer", valueTransformer) + .toString(); + } +} diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/RewindSpec.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/RewindSpec.java index d6a16df9b..1774a6291 100755 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/RewindSpec.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/RewindSpec.java @@ -22,7 +22,10 @@ public class RewindSpec { @JsonProperty("dry_run") private Boolean dryRun; - /** The base timestamp to rewind to. Must be specified. */ + /** + * The base timestamp to rewind to. Exactly one of rewind_timestamp or rewind_point_id must be + * specified. + */ @JsonProperty("rewind_timestamp") private String rewindTimestamp; diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Transformer.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Transformer.java new file mode 100755 index 000000000..d8e30d6eb --- /dev/null +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/Transformer.java @@ -0,0 +1,59 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package com.databricks.sdk.service.pipelines; + +import com.databricks.sdk.support.Generated; +import com.databricks.sdk.support.ToStringer; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; + +/** Specifies how to transform binary data into structured data. */ +@Generated +public class Transformer { + /** Required: the wire format of the data. */ + @JsonProperty("format") + private TransformerFormat format; + + /** */ + @JsonProperty("json_options") + private JsonTransformerOptions jsonOptions; + + public Transformer setFormat(TransformerFormat format) { + this.format = format; + return this; + } + + public TransformerFormat getFormat() { + return format; + } + + public Transformer setJsonOptions(JsonTransformerOptions jsonOptions) { + this.jsonOptions = jsonOptions; + return this; + } + + public JsonTransformerOptions getJsonOptions() { + return jsonOptions; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Transformer that = (Transformer) o; + return Objects.equals(format, that.format) && Objects.equals(jsonOptions, that.jsonOptions); + } + + @Override + public int hashCode() { + return Objects.hash(format, jsonOptions); + } + + @Override + public String toString() { + return new ToStringer(Transformer.class) + .add("format", format) + .add("jsonOptions", jsonOptions) + .toString(); + } +} diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TransformerFormat.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TransformerFormat.java new file mode 100755 index 000000000..06de84ce9 --- /dev/null +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/service/pipelines/TransformerFormat.java @@ -0,0 +1,11 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package com.databricks.sdk.service.pipelines; + +import com.databricks.sdk.support.Generated; + +@Generated +public enum TransformerFormat { + JSON, + STRING, +} diff --git a/tagging.py b/tagging.py index 021e730be..93c5c2d99 100755 --- a/tagging.py +++ b/tagging.py @@ -816,10 +816,82 @@ def preview_tag_infos(packages: List[Package]) -> List[TagInfo]: return [info for info in (get_next_tag_info(package) for package in packages) if info is not None] +def order_tag_infos_by_dependency(tag_infos: List[TagInfo]) -> List[TagInfo]: + """ + Returns ``tag_infos`` in topological order: every package appears + after every sibling it depends on. + """ + if not tag_infos: + return list(tag_infos) + + if any(not info.package.name for info in tag_infos) and len(tag_infos) > 1: + raise Exception("Multiple packages found in legacy mode") + + package_file_path = os.path.join(os.getcwd(), CODEGEN_FILE_NAME) + with open(package_file_path, "r") as file: + codegen = json.load(file) + + name_template = codegen.get("dependency_name_template", "") + dep_patterns = codegen.get("dependency_pattern", {}) + if not name_template or not dep_patterns: + return list(tag_infos) + + by_dep_name: Dict[str, TagInfo] = { + name_template.replace("$PACKAGE", info.package.name): info for info in tag_infos if info.package.name + } + + # Adjacency: path -> set of paths it depends on (within tag_infos). + deps: Dict[str, set] = {info.package.path: set() for info in tag_infos} + for info in tag_infos: + for filename, pattern in dep_patterns.items(): + loc = os.path.join(os.getcwd(), info.package.path, filename) + if not os.path.exists(loc): + continue + with open(loc, "r") as f: + content = f.read() + for dep_name, dep_info in by_dep_name.items(): + if dep_info.package.path == info.package.path: + continue + regex = ( + re.escape(pattern) + .replace(re.escape("$DEPENDENCY"), re.escape(dep_name)) + .replace(re.escape("$VERSION"), Version.PATTERN) + ) + if re.search(regex, content): + deps[info.package.path].add(dep_info.package.path) + + # Stable topological sort: at each step, emit every node whose deps + # are already emitted, alphabetically by package name. Ties broken + # alphabetically so the manifest is reproducible across runs. + emitted: set = set() + ordered: List[TagInfo] = [] + while len(ordered) < len(tag_infos): + ready = sorted( + ( + info + for info in tag_infos + if info.package.path not in emitted and deps[info.package.path].issubset(emitted) + ), + key=lambda info: info.package.name, + ) + if not ready: + remaining = [info.package.name for info in tag_infos if info.package.path not in emitted] + raise Exception(f"Cyclic dependency detected among packages: {remaining}") + for info in ready: + ordered.append(info) + emitted.add(info.package.path) + return ordered + + def push_tags(tag_infos: List[TagInfo]) -> None: """ Creates and pushes tags to the repository. + Tags are emitted in topological order — dependencies before + dependents — so downstream publishing pipelines reading + ``created_tags.json`` can walk it sequentially without re-deriving + the dependency graph. See ``order_tag_infos_by_dependency``. + As a side effect, writes the names of successfully created tags to ``./created_tags.json`` so that workflows triggering this script can discover what was produced (the GitHub Actions workflow uploads this @@ -833,6 +905,7 @@ def push_tags(tag_infos: List[TagInfo]) -> None: exception is re-raised, so recovery-mode runs still surface their output. """ + tag_infos = order_tag_infos_by_dependency(tag_infos) created: List[str] = [] try: for tag_info in tag_infos: