From 74808a1905552e02152d341b8afe8db520de24b6 Mon Sep 17 00:00:00 2001 From: Jonathan Moss <2729151+jwmoss@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:34:57 -0400 Subject: [PATCH] feat: scaffold individual Registry resource modules --- .agents/skills/datatf/SKILL.md | 5 +- CONTRIBUTING.md | 2 +- README.md | 8 ++ docs/module-layouts.md | 90 ++++++++++++ examples/opentofu/README.md | 17 ++- internal/cli/export.go | 81 +++++++++-- internal/cli/resource_modules_test.go | 154 +++++++++++++++++++++ internal/contract/resource_modules.go | 120 ++++++++++++++++ internal/contract/resource_modules_test.go | 60 ++++++++ internal/scaffold/resources.go | 115 +++++++++++++++ internal/scaffold/resources_test.go | 69 +++++++++ scripts/e2e-fake.sh | 103 +++++++------- 12 files changed, 750 insertions(+), 74 deletions(-) create mode 100644 docs/module-layouts.md create mode 100644 internal/cli/resource_modules_test.go create mode 100644 internal/contract/resource_modules.go create mode 100644 internal/contract/resource_modules_test.go create mode 100644 internal/scaffold/resources.go create mode 100644 internal/scaffold/resources_test.go diff --git a/.agents/skills/datatf/SKILL.md b/.agents/skills/datatf/SKILL.md index 4110d0e..045829f 100644 --- a/.agents/skills/datatf/SKILL.md +++ b/.agents/skills/datatf/SKILL.md @@ -2,7 +2,7 @@ name: datatf description: >- Use when an agent must inventory Azure Databricks platform configuration or export it - into the DataTF workspace module contract. + into a supported DataTF Terraform root. --- # DataTF @@ -34,6 +34,9 @@ datatf --json export --profile analytics --scaffold --out ./export Terraform 1.7 or later is required. Use `--module-source` only to select a different compatible module. Local module paths are relative to the generated root, not the current shell directory. +For individual Registry modules without the workspace pattern, read `docs/module-layouts.md`. +Use `--module-layout resources --scaffold`. Keep the existing layout when its state already owns objects. + ## Ownership scope - `workspace` is the default. It includes workspace resources and isolated UC objects bound only to this workspace. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3da2a0..649626c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ formatting, dependency checks, and vet. Go tests use the fake API in `internal/fakews` to check reads, resource selection, and errors. Telemetry tests use local HTTP servers and temporary consent files. They send no production events. -Contract changes also require `make e2e`. That check tests both generated roots: +Contract changes also require `make e2e`. That check tests both scopes in both module layouts: 1. Terraform's [mock provider](https://developer.hashicorp.com/terraform/language/tests/mocking) checks the configuration with the real provider schema. This test excludes import blocks. diff --git a/README.md b/README.md index c6e295e..9ae7089 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,14 @@ datatf export --profile analytics --resources catalogs --name sales --out ./sale A catalog includes its schemas, grants, and bindings. Use `--scope shared` for a shared catalog. Export into a new directory. See [resource selection](docs/resource-selection.md) for details and limits. +To call individual Registry modules without the workspace pattern: + +```sh +datatf export --profile analytics --resources warehouses --module-layout resources --scaffold +``` + +See [module layouts](docs/module-layouts.md) for versions, inputs, and state boundaries. + ## Resource coverage Only the Terraform resources in this table are supported. diff --git a/docs/module-layouts.md b/docs/module-layouts.md new file mode 100644 index 0000000..ad40804 --- /dev/null +++ b/docs/module-layouts.md @@ -0,0 +1,90 @@ +# Module layouts + +DataTF supports two Terraform root layouts. +Both use the same resource selection and ownership rules. +Neither creates Azure workspaces or exports workloads, stored data, or secret values. + +| `--module-layout` | Root calls | Default module version | +| --- | --- | --- | +| `workspace` (default) | `536tech/workspace/databricks` | `0.2.0` | +| `resources` | Individual `536tech` Registry modules | `0.1.0` | + +## Individual modules + +Export one warehouse into a new root: + +```sh +datatf export --profile analytics --resources warehouses --name "Analytics WH" \ + --module-layout resources --scaffold --out ./warehouse +``` + +Omit `--resources` and `--name` to export all supported objects visible in the selected scope. +Use `--scope shared` for shared Unity Catalog objects. Export shared objects once per metastore. + +DataTF writes module blocks, variables, provider settings, versions, inputs, and matching import blocks. +It emits only module types with selected objects. Each type uses `for_each` with +the existing object keys. The root uses one state, not one state per resource module. + +| Module label and input | Registry module | +| --- | --- | +| `catalog` | `536tech/catalog/databricks` | +| `schema` | `536tech/schema/databricks` | +| `storage_credential` | `536tech/storage-credential/databricks` | +| `external_location` | `536tech/external-location/databricks` | +| `workspace_binding` | `536tech/workspace-binding/databricks` | +| `cluster_policy` | `536tech/cluster-policy/databricks` | +| `instance_pool` | `536tech/instance-pool/databricks` | +| `warehouse` | `536tech/sql-warehouse/databricks` | +| `secret_scope` | `536tech/secret-scope/databricks` | +| `service_principal` | `536tech/service-principal/databricks` | + +The `warehouse` label stays consistent with the import addresses. +Its Registry name is `sql-warehouse`. +The modules manage the resources in the [resource matrix](../README.md#resource-coverage). + +## Versions and inputs + +The resource layout pins every emitted module to an exact version. Use `--module-version 0.1.0` +to select that release explicitly. A different version must exist for every selected module and keep +the same inputs and resource addresses. Version ranges are not supported for this layout. + +Generated sources include the `registry.terraform.io` hostname. +This selects the same module Registry in Terraform and OpenTofu. +See the [OpenTofu example](../examples/opentofu/README.md). + +`--module-source` applies only to the workspace layout. +It does not adapt an arbitrary module interface. +The resource layout has no outer module label, so it rejects a nonempty `--root-module` override. + +Resource-layout inputs use the module labels above as keys in `terraform.tfvars`. +Each object contains its module inputs, including its name and supported access rules. +Schema keys use `catalog.schema`. Unity Catalog grants use lists of principals and privileges. +`export.json` retains the existing canonical data structure for tools that read it. + +Without `--scaffold`, DataTF writes inputs, imports, and JSON files only. Use `--scaffold` for a +complete root. References outside the selected resource groups must already exist. +See [resource selection](resource-selection.md) for details. + +## State safety + +Choose the layout before the first import. Changing the layout changes Terraform addresses. +Do not replace an existing root with the other layout or import its objects into another state. +An existing state needs a separately reviewed migration. DataTF does not migrate state. + +For each new root: + +```sh +terraform fmt -check -recursive +terraform init +terraform validate +terraform plan -out=tfplan +terraform show tfplan +``` + +Require imports only: no creates, updates, replacements, or deletes. Apply the saved plan only after +approval. Remove `imports.tf` after import. +Run `terraform plan -detailed-exitcode` and require exit 0. + +Use one workspace state per workspace and one shared state per metastore. +Configure a separate remote backend key for each production root. +Save the existing state and reviewed plan before any migration. diff --git a/examples/opentofu/README.md b/examples/opentofu/README.md index b9f4aa6..5e60d50 100644 --- a/examples/opentofu/README.md +++ b/examples/opentofu/README.md @@ -1,6 +1,6 @@ # OpenTofu -Use DataTF exports with OpenTofu and the same Databricks provider and workspace module. +Use DataTF exports with OpenTofu, the Databricks provider, and individual Registry modules. Install [OpenTofu](https://opentofu.org/docs/intro/install/) alongside DataTF. Keep Git on your `PATH`; the published module downloads from GitHub. No Terraform binary, adapter, or extra DataTF package is required. @@ -15,7 +15,7 @@ Select a saved Databricks profile whose URL matches your workspace. This example databricks auth profiles datatf auth status --profile analytics datatf export --profile analytics --out ./export --scaffold \ - --module-source registry.terraform.io/536tech/workspace/databricks + --module-layout resources cd export tofu fmt -check -recursive tofu init @@ -24,10 +24,9 @@ tofu plan -out=tfplan tofu show tfplan ``` -Name the registry explicitly. OpenTofu resolves an unqualified +The resource layout names `registry.terraform.io` explicitly. OpenTofu resolves an unqualified [module source](https://opentofu.org/docs/language/modules/sources/#module-registry) -through its own registry. -DataTF keeps its default module version, `0.1.0`. +through its own registry. Each generated module block pins version `0.1.0`. The generated root requires version 1.7 or later. Require imports only, with no creates, updates, replacements, or deletes. @@ -41,8 +40,7 @@ An existing Terraform state needs a separate migration review. The local test also needs Go and Bash. Run this command from the DataTF source checkout: ```sh -TF_BIN=tofu E2E_KEEP=1 bash scripts/e2e-fake.sh \ - registry.terraform.io/536tech/workspace/databricks +TF_BIN=tofu E2E_KEEP=1 E2E_LAYOUTS=resources bash scripts/e2e-fake.sh ``` The test exports workspace and shared scopes from the local fake workspace. @@ -54,5 +52,6 @@ Network access is required for downloads. No Docker, Databricks credentials, or cloud resources are required. This test checks module and provider compatibility. It does not test real Databricks permissions. -Verified with OpenTofu 1.12.6, Databricks provider 1.131.0, and workspace module 0.1.0: -27 workspace imports, 8 shared imports, and no changes after import in either scope. +The default workspace pattern uses a separate module interface. Its `0.2.0` release references +child modules without a Registry hostname. This example uses the resource layout instead. +Keep an existing state on its current layout until you review a migration. diff --git a/internal/cli/export.go b/internal/cli/export.go index e74685f..f8d1a03 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "maps" + "regexp" "strings" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -22,6 +23,7 @@ type exportOptions struct { scaffold bool moduleSource string moduleVer string + moduleLayout string resources []string name *string profile string @@ -37,6 +39,7 @@ func newExportCommand(rc *runtime) *cobra.Command { export-report.json for the supported module contract. Use --scaffold to add the Terraform root files. +Use --module-layout resources for individual Registry modules instead of the workspace pattern. Use --resources to read only the selected resource groups: ` + strings.Join(inventory.ResourceNames, "\n ") + ` @@ -59,6 +62,7 @@ from other groups. Use a new output directory for each export.`, Example: ` datatf --profile analytics export --scaffold datatf --profile analytics export --resources catalogs --name sales --scaffold datatf --profile analytics export --resources warehouses --scaffold + datatf --profile analytics export --resources warehouses --module-layout resources --scaffold datatf --profile analytics export --scope shared --resources catalogs --scaffold`, RunE: func(cmd *cobra.Command, args []string) error { return opts.run(cmd, rc) @@ -72,10 +76,11 @@ from other groups. Use a new output directory for each export.`, flags.BoolVar(&opts.allowPartial, "allow-partial", false, "write Terraform even when some reads failed") flags.BoolVar(&opts.scaffold, "scaffold", false, "also write a runnable Terraform root") + flags.StringVar(&opts.moduleLayout, "module-layout", "workspace", "workspace or resources") flags.StringVar(&opts.moduleSource, "module-source", scaffold.DefaultModuleSource, "module source for --scaffold (registry address, Git URL, or local path)") flags.StringVar(&opts.moduleVer, "module-version", "0.2.0", - "module version constraint for --scaffold with a registry source") + "Registry version (resources default 0.1.0; workspace default 0.2.0)") flags.StringSliceVar(&opts.resources, "resources", nil, "limit reads to resource groups (comma-separated)") flags.String("name", "", "select one exact name within one --resources group") @@ -83,16 +88,16 @@ from other groups. Use a new output directory for each export.`, } func (opts *exportOptions) prepare(cmd *cobra.Command) error { + if err := opts.prepareLayout(cmd); err != nil { + return err + } switch opts.scope { case "workspace", "shared": default: return fmt.Errorf("%w: --scope must be workspace or shared", errUsage) } - if opts.rootModule != "" && !hclsyntax.ValidIdentifier(opts.rootModule) { - return fmt.Errorf("%w: --root-module must be a valid Terraform identifier", errUsage) - } - if opts.scaffold && opts.rootModule == "" { - return fmt.Errorf("%w: --scaffold needs a non-empty --root-module", errUsage) + if err := opts.validateRoot(); err != nil { + return err } selected, name, err := resourceSelection(cmd, opts.resources, opts.scope == "shared") if err != nil { @@ -106,6 +111,41 @@ func (opts *exportOptions) prepare(cmd *cobra.Command) error { return nil } +func (opts *exportOptions) validateRoot() error { + if opts.rootModule != "" && !hclsyntax.ValidIdentifier(opts.rootModule) { + return fmt.Errorf("%w: --root-module must be a valid Terraform identifier", errUsage) + } + if opts.scaffold && opts.moduleLayout == "workspace" && opts.rootModule == "" { + return fmt.Errorf("%w: --scaffold needs a non-empty --root-module", errUsage) + } + return nil +} + +func (opts *exportOptions) prepareLayout(cmd *cobra.Command) error { + switch opts.moduleLayout { + case "workspace": + return nil + case "resources": + default: + return fmt.Errorf("%w: --module-layout must be workspace or resources", errUsage) + } + if cmd.Flags().Changed("module-source") { + return fmt.Errorf("%w: --module-source requires --module-layout workspace", errUsage) + } + if cmd.Flags().Changed("root-module") && opts.rootModule != "" { + return fmt.Errorf("%w: --root-module must be empty with --module-layout resources", errUsage) + } + opts.rootModule = "" + if !cmd.Flags().Changed("module-version") { + opts.moduleVer = scaffold.DefaultResourceModuleVersion + } + if !regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`).MatchString(opts.moduleVer) { + return fmt.Errorf("%w: --module-version needs an exact release such as 0.1.0 "+ + "with --module-layout resources", errUsage) + } + return nil +} + func (opts *exportOptions) run(cmd *cobra.Command, rc *runtime) error { if err := opts.prepare(cmd); err != nil { return err @@ -139,19 +179,30 @@ func (opts *exportOptions) files(ex *contract.Export, rep *contract.Report) ( if err != nil { return nil, err } - if opts.scaffold { - more, err := scaffold.Render(scaffold.Options{ - Scope: ex.Scope, Host: rep.Host, Profile: opts.profile, RootModule: opts.rootModule, - ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer, - }) - if err != nil { - return nil, err - } - maps.Copy(files, more) + more, err := opts.scaffoldFiles(ex, rep) + if err != nil { + return nil, err } + maps.Copy(files, more) return files, nil } +func (opts *exportOptions) scaffoldFiles(ex *contract.Export, rep *contract.Report) ( + map[string][]byte, error, +) { + options := scaffold.Options{ + Scope: ex.Scope, Host: rep.Host, Profile: opts.profile, RootModule: opts.rootModule, + ModuleSource: opts.moduleSource, ModuleVersion: opts.moduleVer, + } + if opts.moduleLayout == "resources" { + return scaffold.RenderResources(ex, options, opts.scaffold) + } + if opts.scaffold { + return scaffold.Render(options) + } + return nil, nil +} + func (opts *exportOptions) read(cmd *cobra.Command, rc *runtime) ( *contract.Export, *contract.Report, error, ) { diff --git a/internal/cli/resource_modules_test.go b/internal/cli/resource_modules_test.go new file mode 100644 index 0000000..b555d94 --- /dev/null +++ b/internal/cli/resource_modules_test.go @@ -0,0 +1,154 @@ +package cli + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + + "github.com/536tech/datatf/internal/contract" + "github.com/536tech/datatf/internal/fakews" +) + +func TestExportResourceModules(t *testing.T) { + for _, scope := range []string{"workspace", "shared"} { + t.Run(scope, func(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + out := filepath.Join(t.TempDir(), "out") + code, _, stderr := run(t, "export", "--module-layout", "resources", + "--scope", scope, "--out", out, "--scaffold") + if code != exitOK { + t.Fatalf("exit %d: %s", code, stderr) + } + modules := readResourceModules(t, out) + assertResourceImports(t, out, modules) + want := 10 + if scope == "shared" { + want = 5 + } + if len(modules) != want { + t.Fatalf("module coverage: got %v, want %d", modules, want) + } + }) + } +} + +func readResourceModules(t *testing.T, out string) map[string]bool { + t.Helper() + file, diags := hclsyntax.ParseConfig(readGenerated(t, out, "main.tf"), "main.tf", hcl.InitialPos) + if diags.HasErrors() { + t.Fatal(diags) + } + modules := map[string]bool{} + for _, block := range file.Body.(*hclsyntax.Body).Blocks { + name := block.Labels[0] + modules[name] = true + registryName := strings.ReplaceAll(name, "_", "-") + if name == "warehouse" { + registryName = "sql-warehouse" + } + for key, want := range map[string]string{ + "source": "registry.terraform.io/536tech/" + registryName + "/databricks", + "version": "0.1.0", + } { + value, diags := block.Body.Attributes[key].Expr.Value(nil) + if diags.HasErrors() || value.AsString() != want { + t.Fatalf("%s %s = %v: %v", name, key, value, diags) + } + } + } + return modules +} + +func assertResourceImports(t *testing.T, out string, modules map[string]bool) { + t.Helper() + var ex contract.Export + if err := json.Unmarshal(readGenerated(t, out, "export.json"), &ex); err != nil { + t.Fatal(err) + } + for _, imp := range ex.Imports { + if !modules[imp.Module] { + t.Fatalf("missing module for import: %+v", imp) + } + } + if strings.Contains(string(readGenerated(t, out, "imports.tf")), "module.workspace.") { + t.Fatal("resource imports must not include the workspace pattern") + } +} + +func TestResourceModuleSelection(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + out := filepath.Join(t.TempDir(), "out") + args := []string{"export", "--module-layout", "resources", "--resources", "warehouses", + "--name", "Analytics WH", "--out", out, "--scaffold", "--module-version", "0.1.0"} + code, _, stderr := run(t, args...) + if code != exitOK { + t.Fatalf("exit %d: %s", code, stderr) + } + main := string(readGenerated(t, out, "main.tf")) + if strings.Count(main, `module "`) != 1 || !strings.Contains(main, `module "warehouse"`) { + t.Fatalf("unexpected selected modules: %s", main) + } + vars := string(readGenerated(t, out, "terraform.tfvars")) + if !strings.Contains(vars, `"Analytics WH"`) || !strings.Contains(vars, "warehouse =") { + t.Fatalf("unexpected standalone inputs: %s", vars) + } + assertResourceOverwriteRefused(t, out, args) +} + +func assertResourceOverwriteRefused(t *testing.T, out string, args []string) { + t.Helper() + before := readGenerated(t, out, "main.tf") + code, _, stderr := run(t, args...) + if code != exitErr || !strings.Contains(stderr, "refusing to overwrite") { + t.Fatalf("overwrite exit %d: %s", code, stderr) + } + if string(readGenerated(t, out, "main.tf")) != string(before) { + t.Fatal("existing root changed") + } +} + +func TestResourceModulePartial(t *testing.T) { + srv := fakews.New(t) + isolateAuth(t, srv) + srv.Fail("GET", "/api/2.0/permissions/warehouses/wh1", http.StatusForbidden, "denied") + partial := filepath.Join(t.TempDir(), "partial") + code, _, stderr := run(t, "export", "--module-layout", "resources", "--resources", "warehouses", + "--scaffold", "--out", partial) + if code != exitErr || !strings.Contains(stderr, "partial") { + t.Fatalf("partial exit %d: %s", code, stderr) + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Fatal("partial export wrote files") + } +} + +func TestResourceModuleOptionConflicts(t *testing.T) { + for _, options := range [][]string{ + {"--module-layout", "unknown"}, + {"--module-layout", "resources", "--root-module", "workspace"}, + {"--module-layout", "resources", "--module-source", "../module"}, + {"--module-layout", "resources", "--module-version", "~> 0.1"}, + } { + code, _, stderr := run(t, append([]string{"export", "--scaffold"}, options...)...) + if code != exitUsage || !strings.Contains(stderr, "--module-") { + t.Fatalf("%v: exit %d: %s", options, code, stderr) + } + } +} + +func readGenerated(t *testing.T, out, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join(out, name)) + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/internal/contract/resource_modules.go b/internal/contract/resource_modules.go new file mode 100644 index 0000000..6c7f997 --- /dev/null +++ b/internal/contract/resource_modules.go @@ -0,0 +1,120 @@ +package contract + +import ( + "fmt" + "maps" + "slices" + "strings" + + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" +) + +// ResourceModule contains the selected inputs for one Registry resource module. +type ResourceModule struct { + Name string + Source string + Values cty.Value +} + +type resourceModuleSpec struct { + name string + values any + access map[string]Access + named bool +} + +// ResourceModules adapts the same selection used by Build to individual module inputs. +func ResourceModules(t *Tfvars) ([]ResourceModule, error) { + specs := []resourceModuleSpec{ + {"catalog", t.Catalogs, t.CatalogAccess, true}, + {"schema", resourceSchemas(t), nil, false}, + {"storage_credential", t.StorageCredentials, t.StorageCredentialAccess, true}, + {"external_location", t.ExternalLocations, t.ExternalLocationAccess, true}, + {"workspace_binding", t.WorkspaceBindings, nil, false}, + {"cluster_policy", t.ClusterPolicies, nil, true}, + {"instance_pool", t.InstancePools, nil, true}, + {"warehouse", t.Warehouses, nil, true}, + {"secret_scope", t.SecretScopes, nil, true}, + {"service_principal", t.ServicePrincipals, nil, true}, + } + modules := []ResourceModule{} + for _, spec := range specs { + values, err := resourceValues(spec) + if err != nil { + return nil, fmt.Errorf("build %s module inputs: %w", spec.name, err) + } + if values.LengthInt() == 0 { + continue + } + registryName := strings.ReplaceAll(spec.name, "_", "-") + if spec.name == "warehouse" { + registryName = "sql-warehouse" + } + modules = append(modules, ResourceModule{ + Name: spec.name, Source: "registry.terraform.io/536tech/" + registryName + "/databricks", + Values: values, + }) + } + return modules, nil +} + +func resourceValues(spec resourceModuleSpec) (cty.Value, error) { + values, err := toCty(spec.values) + if err != nil { + return cty.NilVal, err + } + if values.IsNull() { + return cty.EmptyObjectVal, nil + } + objects := values.AsValueMap() + for name, value := range objects { + attributes := value.AsValueMap() + if spec.named { + attributes["name"] = cty.StringVal(name) + } + if spec.access != nil { + attributes["grants"] = resourceGrants(spec.access[name]) + } + objects[name] = cty.ObjectVal(attributes) + } + return cty.ObjectVal(objects), nil +} + +func resourceGrants(access Access) cty.Value { + grants := []cty.Value{} + for _, principal := range slices.Sorted(maps.Keys(access)) { + privileges := []cty.Value{} + for _, privilege := range access[principal] { + privileges = append(privileges, cty.StringVal(privilege)) + } + grants = append(grants, cty.ObjectVal(map[string]cty.Value{ + "principal": cty.StringVal(principal), "privileges": cty.TupleVal(privileges), + })) + } + return cty.TupleVal(grants) +} + +func resourceSchemas(t *Tfvars) map[string]any { + schemas := map[string]any{} + for catalog, names := range t.Schemas { + for _, name := range names { + schemas[catalog+"."+name] = map[string]any{ + "catalog_name": catalog, "name": name, + "storage_root": optionalString(t.SchemaStorageRoots[catalog][name]), + "comment": optionalString(t.SchemaComments[catalog][name]), + "grants": ctyjson.SimpleJSONValue{ + Value: resourceGrants(t.SchemaAccess[catalog][name]), + }, + } + } + } + return schemas +} + +func optionalString(value string) any { + if value == "" { + return nil + } + return value +} diff --git a/internal/contract/resource_modules_test.go b/internal/contract/resource_modules_test.go new file mode 100644 index 0000000..ea82c45 --- /dev/null +++ b/internal/contract/resource_modules_test.go @@ -0,0 +1,60 @@ +package contract + +import ( + "strings" + "testing" + + "github.com/zclconf/go-cty/cty" +) + +func TestResourceModuleInputs(t *testing.T) { + const name = `sales"${1+1}` + tfvars := newTfvars(ScopeWorkspace) + tfvars.Catalogs[name] = CatalogSettings{IsolationMode: "ISOLATED", Owner: "owner"} + tfvars.CatalogAccess[name] = Access{"z": {"USE_CATALOG"}, "a": {"USE_CATALOG"}} + tfvars.Schemas[name] = []string{"bronze"} + tfvars.SchemaAccess[name] = map[string]Access{"bronze": {"analysts": {"USE_SCHEMA"}}} + tfvars.WorkspaceBindings["binding"] = WorkspaceBindingSettings{WorkspaceID: 9007199254740993} + modules, err := ResourceModules(tfvars) + if err != nil { + t.Fatal(err) + } + values := map[string]cty.Value{} + for _, module := range modules { + values[module.Name] = module.Values + } + catalog := values["catalog"].GetAttr(name) + assertModuleValue(t, catalog.GetAttr("name"), cty.StringVal(name)) + assertModuleValue(t, catalog.GetAttr("grants").Index(cty.NumberIntVal(0)).GetAttr("principal"), + cty.StringVal("a")) + schema := values["schema"].GetAttr(name + ".bronze") + assertModuleValue(t, schema.GetAttr("catalog_name"), cty.StringVal(name)) + assertModuleValue(t, schema.GetAttr("grants").Length(), cty.NumberIntVal(1)) + if !schema.GetAttr("storage_root").IsNull() || !schema.GetAttr("comment").IsNull() { + t.Fatal("missing optional schema values must stay null") + } + if !values["workspace_binding"].GetAttr("binding").GetAttr("workspace_id").RawEquals( + cty.NumberIntVal(9007199254740993)) { + t.Fatal("workspace ID lost numeric precision") + } +} + +func assertModuleValue(t *testing.T, got, want cty.Value) { + t.Helper() + if !got.RawEquals(want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestResourceModulesEmptyAndInvalid(t *testing.T) { + modules, err := ResourceModules(newTfvars(ScopeShared)) + if err != nil || len(modules) != 0 { + t.Fatalf("empty selection: %v %v", modules, err) + } + tfvars := newTfvars(ScopeWorkspace) + tfvars.ClusterPolicies["invalid"] = ClusterPolicySettings{Definition: make(chan int)} + _, err = ResourceModules(tfvars) + if err == nil || !strings.Contains(err.Error(), "cluster_policy module inputs") { + t.Fatalf("missing rendering context: %v", err) + } +} diff --git a/internal/scaffold/resources.go b/internal/scaffold/resources.go new file mode 100644 index 0000000..eaf6a11 --- /dev/null +++ b/internal/scaffold/resources.go @@ -0,0 +1,115 @@ +package scaffold + +import ( + "maps" + "slices" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" + + "github.com/536tech/datatf/internal/contract" +) + +// DefaultResourceModuleVersion pins the tested individual Registry modules. +const DefaultResourceModuleVersion = "0.1.0" + +// RenderResources renders module inputs and, when requested, their runnable root. +func RenderResources(ex *contract.Export, opts Options, root bool) (map[string][]byte, error) { + modules, err := contract.ResourceModules(ex.Tfvars) + if err != nil { + return nil, err + } + inputs := make([]contract.Variable, 0, len(modules)) + for _, module := range modules { + inputs = append(inputs, contract.Variable{ + Name: module.Name, Value: ctyjson.SimpleJSONValue{Value: module.Values}, + }) + } + tfvars, err := contract.RenderVariables(inputs, contract.Header(ex.Scope, opts.Host)) + if err != nil { + return nil, err + } + files := map[string][]byte{"terraform.tfvars": tfvars} + if !root { + return files, nil + } + main, variables := resourceRoot(modules, opts.ModuleVersion) + maps.Copy(files, map[string][]byte{ + "main.tf": main, "variables.tf": variables, + "providers.tf": providerFile(opts), "versions.tf": []byte(versionsTF), + "README.md": []byte(readmeMD + resourceReadmeMD), + }) + return files, nil +} + +const resourceReadmeMD = ` +## Individual resource modules + +This root calls individual Registry modules, without an outer workspace module. +The module blocks pin their versions. terraform.tfvars uses the module labels as input names. +export.json retains the canonical DataTF data structure, not the resource-module input shape. + +Keep this layout after import. A layout change requires a reviewed state migration. +` + +func resourceRoot(modules []contract.ResourceModule, version string) ([]byte, []byte) { + main, variables := hclwrite.NewEmptyFile(), hclwrite.NewEmptyFile() + for _, module := range modules { + body := main.Body().AppendNewBlock("module", []string{module.Name}).Body() + body.SetAttributeValue("source", cty.StringVal(module.Source)) + body.SetAttributeValue("version", cty.StringVal(version)) + body.SetAttributeTraversal("for_each", traversal("var", module.Name)) + for _, name := range resourceInputNames(module.Values) { + body.SetAttributeRaw(name, hclwrite.TokensForFunctionCall("try", + hclwrite.TokensForTraversal(traversal("each", "value", name)), + hclwrite.TokensForValue(cty.NullVal(cty.DynamicPseudoType)), + )) + } + resourceDependencies(body, module.Name, modules) + main.Body().AppendNewline() + variable := variables.Body().AppendNewBlock("variable", []string{module.Name}).Body() + variable.SetAttributeValue("description", cty.StringVal("Inputs for "+module.Source+".")) + // Resource objects can have different policy JSON shapes; the child validates its inputs. + variable.SetAttributeTraversal("type", traversal("any")) + variable.SetAttributeValue("default", cty.EmptyObjectVal) + variables.Body().AppendNewline() + } + return main.Bytes(), variables.Bytes() +} + +func resourceInputNames(values cty.Value) []string { + names := map[string]bool{} + for _, value := range values.AsValueMap() { + for name := range value.AsValueMap() { + names[name] = true + } + } + return slices.Sorted(maps.Keys(names)) +} + +func resourceDependencies(body *hclwrite.Body, name string, modules []contract.ResourceModule) { + dependencies := map[string][]string{ + "schema": {"catalog"}, + "external_location": {"storage_credential"}, + "workspace_binding": {"catalog", "storage_credential", "external_location"}, + } + tokens := []hclwrite.Tokens{} + for _, module := range modules { + if slices.Contains(dependencies[name], module.Name) { + tokens = append(tokens, hclwrite.TokensForTraversal(traversal("module", module.Name))) + } + } + if len(tokens) > 0 { + body.SetAttributeRaw("depends_on", hclwrite.TokensForTuple(tokens)) + } +} + +func traversal(root string, attributes ...string) hcl.Traversal { + value := hcl.Traversal{hcl.TraverseRoot{Name: root}} + for _, name := range attributes { + value = append(value, hcl.TraverseAttr{Name: name}) + } + return value +} diff --git a/internal/scaffold/resources_test.go b/internal/scaffold/resources_test.go new file mode 100644 index 0000000..cea9171 --- /dev/null +++ b/internal/scaffold/resources_test.go @@ -0,0 +1,69 @@ +package scaffold + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/536tech/datatf/internal/contract" +) + +func TestResourceRootDeterminismAndProfile(t *testing.T) { + ex := &contract.Export{Scope: contract.ScopeWorkspace, Tfvars: &contract.Tfvars{ + Catalogs: map[string]contract.CatalogSettings{ + `b"${1+1}`: {IsolationMode: "ISOLATED", Owner: "owner"}, + "a": {IsolationMode: "ISOLATED", Owner: "owner", Comment: "a catalog"}, + }, + }} + opts := Options{Host: "https://workspace.example.com", Profile: `profile"${1+1}`, + ModuleVersion: DefaultResourceModuleVersion} + before, err := json.Marshal(ex) + if err != nil { + t.Fatal(err) + } + files, err := RenderResources(ex, opts, true) + if err != nil { + t.Fatal(err) + } + assertResourceRenderStable(t, ex, opts, files) + provider := parseBlock(t, files["providers.tf"]).Body + assertAttribute(t, provider, "profile", opts.Profile) + assertAttribute(t, provider, "host", opts.Host) + after, err := json.Marshal(ex) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("rendering changed the canonical export") + } +} + +func assertResourceRenderStable(t *testing.T, ex *contract.Export, opts Options, + want map[string][]byte, +) { + t.Helper() + for range 10 { + got, err := RenderResources(ex, opts, true) + if err != nil { + t.Fatal(err) + } + for name, data := range want { + if !bytes.Equal(data, got[name]) { + t.Fatalf("%s is not deterministic", name) + } + } + } +} + +func TestResourceRootEmptyAndInputsOnly(t *testing.T) { + ex := &contract.Export{Scope: contract.ScopeShared, Tfvars: &contract.Tfvars{}} + opts := Options{ModuleVersion: DefaultResourceModuleVersion} + files, err := RenderResources(ex, opts, false) + if err != nil || len(files) != 1 || files["terraform.tfvars"] == nil { + t.Fatalf("inputs-only files: %v, %v", files, err) + } + files, err = RenderResources(ex, opts, true) + if err != nil { + t.Fatal(err) + } + if len(files["main.tf"]) != 0 || len(files["variables.tf"]) != 0 { + t.Fatal("empty selection emitted module blocks or variables") + } +} diff --git a/scripts/e2e-fake.sh b/scripts/e2e-fake.sh index 50c3471..974f25b 100755 --- a/scripts/e2e-fake.sh +++ b/scripts/e2e-fake.sh @@ -47,53 +47,60 @@ export DATABRICKS_CONFIG_FILE="$WORK/none.cfg" export DATABRICKS_DISCOVERY_URL="http://$ADDR/.well-known/oauth-authorization-server" unset DATABRICKS_CONFIG_PROFILE || true -for scope in workspace shared; do - OUT="$WORK/$scope" - "$WORK/datatf" export --quiet --scope "$scope" --out "$OUT" \ - --scaffold --module-source "$MODULES" - ( - cd "$OUT" - "$TF_BIN" fmt -check -recursive - "$TF_BIN" init -input=false -no-color >/dev/null - "$TF_BIN" validate -no-color - # Mock tests check configuration; the real provider checks imports below. - mv imports.tf "$WORK/$scope-imports.tf" - mkdir tests - cp "$ROOT/tests/export.tftest.hcl" tests/ - "$TF_BIN" test -no-color - mv "$WORK/$scope-imports.tf" imports.tf - set +e - "$TF_BIN" plan -input=false -no-color -detailed-exitcode -out=tfplan \ - >plan-create.txt 2>plan.err - code=$? - set -e - if [ "$code" -eq 1 ]; then - echo "$TF_BIN plan failed for $scope:" - cat plan.err - KEEP=1 - exit 1 +read -r -a layouts <<<"${E2E_LAYOUTS:-workspace resources}" +for layout in "${layouts[@]}"; do + for scope in workspace shared; do + OUT="$WORK/$layout/$scope" + module_args=(--module-layout "$layout") + if [[ "$layout" == workspace ]]; then + module_args+=(--module-source "$MODULES") fi - "$TF_BIN" show -no-color tfplan >plan.txt - grep -E "Plan: .*to import" plan.txt || { - echo "no import summary in plan ($scope)" - tail -40 plan.txt - exit 1 - } - if ! grep -qE "Plan: [1-9][0-9]* to import, 0 to add, 0 to change, 0 to destroy" plan.txt; then - echo "plan for $scope is not import-only (exit $code):" - grep -nE "will be|must be" plan.txt | grep -v "will be imported" || true - KEEP=1 - exit 1 - fi - echo "OK: $scope -> $(grep -E '^Plan:' plan.txt)" - "$TF_BIN" apply -input=false -no-color tfplan >apply.txt - mv imports.tf "$WORK/$scope-imports.tf" - if ! "$TF_BIN" plan -input=false -no-color -detailed-exitcode >plan-after.txt 2>&1; then - echo "$TF_BIN plan after import is not clean ($scope):" - cat plan-after.txt - KEEP=1 - exit 1 - fi - echo "OK: $scope -> no changes after import" - ) + "$WORK/datatf" export --quiet --scope "$scope" --out "$OUT" \ + --scaffold "${module_args[@]}" + ( + cd "$OUT" + "$TF_BIN" fmt -check -recursive + "$TF_BIN" init -input=false -no-color >/dev/null + "$TF_BIN" validate -no-color + # Mock tests check configuration; the real provider checks imports below. + mv imports.tf "$WORK/$layout-$scope-imports.tf" + mkdir tests + cp "$ROOT/tests/export.tftest.hcl" tests/ + "$TF_BIN" test -no-color + mv "$WORK/$layout-$scope-imports.tf" imports.tf + set +e + "$TF_BIN" plan -input=false -no-color -detailed-exitcode -out=tfplan \ + >plan-create.txt 2>plan.err + code=$? + set -e + if [ "$code" -eq 1 ]; then + echo "$TF_BIN plan failed for $scope:" + cat plan.err + KEEP=1 + exit 1 + fi + "$TF_BIN" show -no-color tfplan >plan.txt + grep -E "Plan: .*to import" plan.txt || { + echo "no import summary in plan ($scope)" + tail -40 plan.txt + exit 1 + } + if ! grep -qE "Plan: [1-9][0-9]* to import, 0 to add, 0 to change, 0 to destroy" plan.txt; then + echo "plan for $scope is not import-only (exit $code):" + grep -nE "will be|must be" plan.txt | grep -v "will be imported" || true + KEEP=1 + exit 1 + fi + echo "OK: $layout/$scope -> $(grep -E '^Plan:' plan.txt)" + "$TF_BIN" apply -input=false -no-color tfplan >apply.txt + mv imports.tf "$WORK/$layout-$scope-imports.tf" + if ! "$TF_BIN" plan -input=false -no-color -detailed-exitcode >plan-after.txt 2>&1; then + echo "$TF_BIN plan after import is not clean ($scope):" + cat plan-after.txt + KEEP=1 + exit 1 + fi + echo "OK: $layout/$scope -> no changes after import" + ) + done done