feat: scaffold individual Registry resource modules

.agents/skills/datatf/SKILL.md

@@ -2,7 +2,7 @@
 name: datatf
 description: >-
   Use when an agent must inventory Azure Databricks platform configuration or export it
-  into the DataTF workspace module contract.
+  into a supported DataTF Terraform root.
 ---
 
 # DataTF
@@ -34,6 +34,9 @@ datatf --json export --profile analytics --scaffold --out ./export
 Terraform 1.7 or later is required. Use `--module-source` only to select a different compatible module.
 Local module paths are relative to the generated root, not the current shell directory.
 
+For individual Registry modules without the workspace pattern, read `docs/module-layouts.md`.
+Use `--module-layout resources --scaffold`. Keep the existing layout when its state already owns objects.
+
 ## Ownership scope
 
 - `workspace` is the default. It includes workspace resources and isolated UC objects bound only to this workspace.

CONTRIBUTING.md

@@ -20,7 +20,7 @@ formatting, dependency checks, and vet.
 
 Go tests use the fake API in `internal/fakews` to check reads, resource selection, and errors.
 Telemetry tests use local HTTP servers and temporary consent files. They send no production events.
-Contract changes also require `make e2e`. That check tests both generated roots:
+Contract changes also require `make e2e`. That check tests both scopes in both module layouts:
 
 1. Terraform's [mock provider](https://developer.hashicorp.com/terraform/language/tests/mocking)
    checks the configuration with the real provider schema. This test excludes import blocks.

README.md

@@ -73,6 +73,14 @@ datatf export --profile analytics --resources catalogs --name sales --out ./sale
 A catalog includes its schemas, grants, and bindings. Use `--scope shared` for a shared catalog.
 Export into a new directory. See [resource selection](docs/resource-selection.md) for details and limits.
 
+To call individual Registry modules without the workspace pattern:
+
+```sh
+datatf export --profile analytics --resources warehouses --module-layout resources --scaffold
+```
+
+See [module layouts](docs/module-layouts.md) for versions, inputs, and state boundaries.
+
 ## Resource coverage
 
 Only the Terraform resources in this table are supported.

docs/module-layouts.md

@@ -0,0 +1,90 @@
+# Module layouts
+
+DataTF supports two Terraform root layouts.
+Both use the same resource selection and ownership rules.
+Neither creates Azure workspaces or exports workloads, stored data, or secret values.
+
+| `--module-layout` | Root calls | Default module version |
+| --- | --- | --- |
+| `workspace` (default) | `536tech/workspace/databricks` | `0.2.0` |
+| `resources` | Individual `536tech` Registry modules | `0.1.0` |
+
+## Individual modules
+
+Export one warehouse into a new root:
+
+```sh
+datatf export --profile analytics --resources warehouses --name "Analytics WH" \
+  --module-layout resources --scaffold --out ./warehouse
+```
+
+Omit `--resources` and `--name` to export all supported objects visible in the selected scope.
+Use `--scope shared` for shared Unity Catalog objects. Export shared objects once per metastore.
+
+DataTF writes module blocks, variables, provider settings, versions, inputs, and matching import blocks.
+It emits only module types with selected objects. Each type uses `for_each` with
+the existing object keys. The root uses one state, not one state per resource module.
+
+| Module label and input | Registry module |
+| --- | --- |
+| `catalog` | `536tech/catalog/databricks` |
+| `schema` | `536tech/schema/databricks` |
+| `storage_credential` | `536tech/storage-credential/databricks` |
+| `external_location` | `536tech/external-location/databricks` |
+| `workspace_binding` | `536tech/workspace-binding/databricks` |
+| `cluster_policy` | `536tech/cluster-policy/databricks` |
+| `instance_pool` | `536tech/instance-pool/databricks` |
+| `warehouse` | `536tech/sql-warehouse/databricks` |
+| `secret_scope` | `536tech/secret-scope/databricks` |
+| `service_principal` | `536tech/service-principal/databricks` |
+
+The `warehouse` label stays consistent with the import addresses.
+Its Registry name is `sql-warehouse`.
+The modules manage the resources in the [resource matrix](../README.md#resource-coverage).
+
+## Versions and inputs
+
+The resource layout pins every emitted module to an exact version. Use `--module-version 0.1.0`
+to select that release explicitly. A different version must exist for every selected module and keep
+the same inputs and resource addresses. Version ranges are not supported for this layout.
+
+Generated sources include the `registry.terraform.io` hostname.
+This selects the same module Registry in Terraform and OpenTofu.
+See the [OpenTofu example](../examples/opentofu/README.md).
+
+`--module-source` applies only to the workspace layout.
+It does not adapt an arbitrary module interface.
+The resource layout has no outer module label, so it rejects a nonempty `--root-module` override.
+
+Resource-layout inputs use the module labels above as keys in `terraform.tfvars`.
+Each object contains its module inputs, including its name and supported access rules.
+Schema keys use `catalog.schema`. Unity Catalog grants use lists of principals and privileges.
+`export.json` retains the existing canonical data structure for tools that read it.
+
+Without `--scaffold`, DataTF writes inputs, imports, and JSON files only. Use `--scaffold` for a
+complete root. References outside the selected resource groups must already exist.
+See [resource selection](resource-selection.md) for details.
+
+## State safety
+
+Choose the layout before the first import. Changing the layout changes Terraform addresses.
+Do not replace an existing root with the other layout or import its objects into another state.
+An existing state needs a separately reviewed migration. DataTF does not migrate state.
+
+For each new root:
+
+```sh
+terraform fmt -check -recursive
+terraform init
+terraform validate
+terraform plan -out=tfplan
+terraform show tfplan
+```
+
+Require imports only: no creates, updates, replacements, or deletes. Apply the saved plan only after
+approval. Remove `imports.tf` after import.
+Run `terraform plan -detailed-exitcode` and require exit 0.
+
+Use one workspace state per workspace and one shared state per metastore.
+Configure a separate remote backend key for each production root.
+Save the existing state and reviewed plan before any migration.

examples/opentofu/README.md

@@ -1,6 +1,6 @@
 # OpenTofu
 
-Use DataTF exports with OpenTofu and the same Databricks provider and workspace module.
+Use DataTF exports with OpenTofu, the Databricks provider, and individual Registry modules.
 Install [OpenTofu](https://opentofu.org/docs/intro/install/) alongside DataTF.
 Keep Git on your `PATH`; the published module downloads from GitHub.
 No Terraform binary, adapter, or extra DataTF package is required.
@@ -15,7 +15,7 @@ Select a saved Databricks profile whose URL matches your workspace. This example
 databricks auth profiles
 datatf auth status --profile analytics
 datatf export --profile analytics --out ./export --scaffold \
-  --module-source registry.terraform.io/536tech/workspace/databricks
+  --module-layout resources
 cd export
 tofu fmt -check -recursive
 tofu init
@@ -24,10 +24,9 @@ tofu plan -out=tfplan
 tofu show tfplan
 ```
 
-Name the registry explicitly. OpenTofu resolves an unqualified
+The resource layout names `registry.terraform.io` explicitly. OpenTofu resolves an unqualified
 [module source](https://opentofu.org/docs/language/modules/sources/#module-registry)
-through its own registry.
-DataTF keeps its default module version, `0.1.0`.
+through its own registry. Each generated module block pins version `0.1.0`.
 The generated root requires version 1.7 or later.
 
 Require imports only, with no creates, updates, replacements, or deletes.
@@ -41,8 +40,7 @@ An existing Terraform state needs a separate migration review.
 The local test also needs Go and Bash. Run this command from the DataTF source checkout:
 
 ```sh
-TF_BIN=tofu E2E_KEEP=1 bash scripts/e2e-fake.sh \
-  registry.terraform.io/536tech/workspace/databricks
+TF_BIN=tofu E2E_KEEP=1 E2E_LAYOUTS=resources bash scripts/e2e-fake.sh
 ```
 
 The test exports workspace and shared scopes from the local fake workspace.
@@ -54,5 +52,6 @@ Network access is required for downloads.
 No Docker, Databricks credentials, or cloud resources are required.
 This test checks module and provider compatibility. It does not test real Databricks permissions.
 
-Verified with OpenTofu 1.12.6, Databricks provider 1.131.0, and workspace module 0.1.0:
-27 workspace imports, 8 shared imports, and no changes after import in either scope.
+The default workspace pattern uses a separate module interface. Its `0.2.0` release references
+child modules without a Registry hostname. This example uses the resource layout instead.
+Keep an existing state on its current layout until you review a migration.

internal/cli/export.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"fmt"
 	"maps"
+	"regexp"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2/hclsyntax"
@@ -22,6 +23,7 @@ type exportOptions struct {
 	scaffold     bool
 	moduleSource string
 	moduleVer    string
+	moduleLayout string
 	resources    []string
 	name         *string
 	profile      string
@@ -37,6 +39,7 @@ func newExportCommand(rc *runtime) *cobra.Command {
 export-report.json for the supported module contract.
 
 Use --scaffold to add the Terraform root files.
+Use --module-layout resources for individual Registry modules instead of the workspace pattern.
 Use --resources to read only the selected resource groups:
   ` + strings.Join(inventory.ResourceNames, "\n  ") + `
 
@@ -59,6 +62,7 @@ from other groups. Use a new output directory for each export.`,
 		Example: `  datatf --profile analytics export --scaffold
   datatf --profile analytics export --resources catalogs --name sales --scaffold
   datatf --profile analytics export --resources warehouses --scaffold
+  datatf --profile analytics export --resources warehouses --module-layout resources --scaffold
   datatf --profile analytics export --scope shared --resources catalogs --scaffold`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return opts.run(cmd, rc)
@@ -72,27 +76,28 @@ from other groups. Use a new output directory for each export.`,
 	flags.BoolVar(&opts.allowPartial, "allow-partial", false,
 		"write Terraform even when some reads failed")
 	flags.BoolVar(&opts.scaffold, "scaffold", false, "also write a runnable Terraform root")
+	flags.StringVar(&opts.moduleLayout, "module-layout", "workspace", "workspace or resources")
 	flags.StringVar(&opts.moduleSource, "module-source", scaffold.DefaultModuleSource,
 		"module source for --scaffold (registry address, Git URL, or local path)")
 	flags.StringVar(&opts.moduleVer, "module-version", "0.2.0",
-		"module version constraint for --scaffold with a registry source")
+		"Registry version (resources default 0.1.0; workspace default 0.2.0)")
 	flags.StringSliceVar(&opts.resources, "resources", nil,
 		"limit reads to resource groups (comma-separated)")
 	flags.String("name", "", "select one exact name within one --resources group")
 	return cmd
 }
 
 func (opts *exportOptions) prepare(cmd *cobra.Command) error {
+	if err := opts.prepareLayout(cmd); err != nil {
+		return err
+	}
 	switch opts.scope {
 	case "workspace", "shared":
 	default:
 		return fmt.Errorf("%w: --scope must be workspace or shared", errUsage)
 	}
-	if opts.rootModule != "" && !hclsyntax.ValidIdentifier(opts.rootModule) {
-		return fmt.Errorf("%w: --root-module must be a valid Terraform identifier", errUsage)
-	}
-	if opts.scaffold && opts.rootModule == "" {
-		return fmt.Errorf("%w: --scaffold needs a non-empty --root-module", errUsage)
+	if err := opts.validateRoot(); err != nil {
+		return err
 	}
 	selected, name, err := resourceSelection(cmd, opts.resources, opts.scope == "shared")
 	if err != nil {
@@ -106,6 +111,41 @@ func (opts *exportOptions) prepare(cmd *cobra.Command) error {
 	return nil
 }
 
+func (opts *exportOptions) validateRoot() error {
+	if opts.rootModule != "" && !hclsyntax.ValidIdentifier(opts.rootModule) {
+		return fmt.Errorf("%w: --root-module must be a valid Terraform identifier", errUsage)
+	}
+	if opts.scaffold && opts.moduleLayout == "workspace" && opts.rootModule == "" {
+		return fmt.Errorf("%w: --scaffold needs a non-empty --root-module", errUsage)
+	}
+	return nil
+}
+
+func (opts *exportOptions) prepareLayout(cmd *cobra.Command) error {
+	switch opts.moduleLayout {
+	case "workspace":
+		return nil
+	case "resources":
+	default:
+		return fmt.Errorf("%w: --module-layout must be workspace or resources", errUsage)
+	}
+	if cmd.Flags().Changed("module-source") {
+		return fmt.Errorf("%w: --module-source requires --module-layout workspace", errUsage)
+	}
+	if cmd.Flags().Changed("root-module") && opts.rootModule != "" {
+		return fmt.Errorf("%w: --root-module must be empty with --module-layout resources", errUsage)
+	}
+	opts.rootModule = ""
+	if !cmd.Flags().Changed("module-version") {
+		opts.moduleVer = scaffold.DefaultResourceModuleVersion
+	}
+	if !regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`).MatchString(opts.moduleVer) {
+		return fmt.Errorf("%w: --module-version needs an exact release such as 0.1.0 "+
+			"with --module-layout resources", errUsage)
+	}
+	return nil
+}
+
 func (opts *exportOptions) run(cmd *cobra.Command, rc *runtime) error {
 	if err := opts.prepare(cmd); err != nil {
 		return err
@@ -139,19 +179,30 @@ func (opts *exportOptions) files(ex *contract.Export, rep *contract.Report) (
 	if err != nil {
 		return nil, err
 	}
-	if opts.scaffold {
-		more, err := scaffold.Render(scaffold.Options{
-			Scope: ex.Scope, Host: rep.Host, Profile: opts.profile, RootModule: opts.rootModule,
-			ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer,
-		})
-		if err != nil {
-			return nil, err
-		}
-		maps.Copy(files, more)
+	more, err := opts.scaffoldFiles(ex, rep)
+	if err != nil {
+		return nil, err
 	}
+	maps.Copy(files, more)
 	return files, nil
 }
 
+func (opts *exportOptions) scaffoldFiles(ex *contract.Export, rep *contract.Report) (
+	map[string][]byte, error,
+) {
+	options := scaffold.Options{
+		Scope: ex.Scope, Host: rep.Host, Profile: opts.profile, RootModule: opts.rootModule,
+		ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer,
+	}
+	if opts.moduleLayout == "resources" {
+		return scaffold.RenderResources(ex, options, opts.scaffold)
+	}
+	if opts.scaffold {
+		return scaffold.Render(options)
+	}
+	return nil, nil
+}
+
 func (opts *exportOptions) read(cmd *cobra.Command, rc *runtime) (
 	*contract.Export, *contract.Report, error,
 ) {