From a290fe343aacc19b9b7a1e0e1634221367bb4ac4 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 27 Feb 2026 11:17:21 +0530 Subject: [PATCH] Support volume_path in direct deploy engine Signed-off-by: Varun Deep Saini --- .../debug/refschema_volume_path/out.test.toml | 5 + .../refschema_volume_path/out.volume_path.txt | 1 + .../debug/refschema_volume_path/output.txt | 5 + .../bundle/debug/refschema_volume_path/script | 2 + .../debug/refschema_volume_path/test.toml | 5 + .../bundle/migrate/basic/out.new_state.json | 1 + .../bundle/migrate/basic/out.plan_update.json | 1 + .../bundle/migrate/grants/out.new_state.json | 1 + .../bundle/python/volumes-support/output.txt | 2 + acceptance/bundle/refschema/out.fields.txt | 1 + .../resource_deps/bad_syntax/output.txt | 2 + .../non_existent_field/out.deploy.direct.txt | 2 +- .../non_existent_field/out.plan.direct.txt | 2 +- .../non_existent_field/output.txt | 2 + .../remote_field_storage_location/output.txt | 2 + .../databricks.yml.tmpl | 20 +++ .../out.deploy.direct.txt | 6 + .../out.deploy.requests.direct.json | 29 +++++ .../out.deploy.requests.terraform.json | 41 ++++++ .../out.deploy.terraform.txt | 6 + .../out.destroy.direct.txt | 19 +++ .../out.destroy.terraform.txt | 19 +++ .../out.destroy_requests.direct.json | 15 +++ .../out.destroy_requests.terraform.json | 15 +++ .../out.plan_after_deploy.direct.txt | 1 + .../out.plan_after_deploy.terraform.txt | 1 + .../remote_field_volume_path/out.test.toml | 10 ++ .../remote_field_volume_path/output.txt | 36 ++++++ .../remote_field_volume_path/script | 15 +++ .../remote_field_volume_path/test.toml | 10 ++ .../databricks.yml.tmpl | 15 +++ .../out.plan.direct.json | 37 ++++++ .../out.test.toml | 5 + .../remote_field_volume_path_plan/output.txt | 50 ++++++++ .../remote_field_volume_path_plan/script | 6 + .../remote_field_volume_path_plan/test.toml | 5 + .../grants/volumes/out.plan1.direct.json | 1 + .../grants/volumes/out.plan2.direct.json | 1 + .../volumes/change-comment/output.txt | 3 + .../volumes/change-name/out.plan.direct.json | 8 ++ .../resources/volumes/change-name/output.txt | 1 + .../volumes/change-schema-name/output.txt | 1 + .../out.plan_create.direct.json | 1 + .../validate/presets_name_prefix/output.txt | 3 + .../out.with_empty_prefix.json | 1 + .../out.with_nil_prefix.json | 1 + .../out.with_static_prefix.json | 1 + .../out.without_presets.json | 1 + .../mutator/python/apply_python_output.go | 30 ++++- .../python/apply_python_output_test.go | 25 +++- .../config/mutator/python/python_mutator.go | 75 +++++++++++ .../mutator/python/python_mutator_test.go | 32 +++++ .../capture_schema_dependency.go | 14 ++- .../capture_schema_dependency_test.go | 12 +- bundle/config/resources/volume.go | 2 + .../deploy/terraform/tfdyn/convert_volume.go | 6 + .../terraform/tfdyn/convert_volume_test.go | 1 + bundle/direct/dresources/volume.go | 118 +++++++++++++++--- bundle/direct/dresources/volume_test.go | 78 ++++++++++++ 59 files changed, 781 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/debug/refschema_volume_path/out.test.toml create mode 100644 acceptance/bundle/debug/refschema_volume_path/out.volume_path.txt create mode 100644 acceptance/bundle/debug/refschema_volume_path/output.txt create mode 100755 acceptance/bundle/debug/refschema_volume_path/script create mode 100644 acceptance/bundle/debug/refschema_volume_path/test.toml create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.direct.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.direct.json create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.terraform.json create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.direct.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.direct.json create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.terraform.json create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.direct.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.terraform.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/output.txt create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/script create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path/test.toml create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path_plan/databricks.yml.tmpl create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.plan.direct.json create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.test.toml create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path_plan/output.txt create mode 100755 acceptance/bundle/resource_deps/remote_field_volume_path_plan/script create mode 100644 acceptance/bundle/resource_deps/remote_field_volume_path_plan/test.toml create mode 100644 bundle/direct/dresources/volume_test.go diff --git a/acceptance/bundle/debug/refschema_volume_path/out.test.toml b/acceptance/bundle/debug/refschema_volume_path/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/debug/refschema_volume_path/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/debug/refschema_volume_path/out.volume_path.txt b/acceptance/bundle/debug/refschema_volume_path/out.volume_path.txt new file mode 100644 index 0000000000..bf591b5cea --- /dev/null +++ b/acceptance/bundle/debug/refschema_volume_path/out.volume_path.txt @@ -0,0 +1 @@ +resources.volumes.*.volume_path string ALL diff --git a/acceptance/bundle/debug/refschema_volume_path/output.txt b/acceptance/bundle/debug/refschema_volume_path/output.txt new file mode 100644 index 0000000000..f6d7f5c8d8 --- /dev/null +++ b/acceptance/bundle/debug/refschema_volume_path/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle debug refschema + +>>> cat out.volume_path.txt +resources.volumes.*.volume_path string ALL diff --git a/acceptance/bundle/debug/refschema_volume_path/script b/acceptance/bundle/debug/refschema_volume_path/script new file mode 100755 index 0000000000..7b120781b3 --- /dev/null +++ b/acceptance/bundle/debug/refschema_volume_path/script @@ -0,0 +1,2 @@ +trace $CLI bundle debug refschema | grep '^resources.volumes.\*.volume_path[[:space:]]\+string[[:space:]]\+ALL$' > out.volume_path.txt +trace cat out.volume_path.txt diff --git a/acceptance/bundle/debug/refschema_volume_path/test.toml b/acceptance/bundle/debug/refschema_volume_path/test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/debug/refschema_volume_path/test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index de537f2f4b..2771be3820 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -87,6 +87,7 @@ "catalog_name": "mycat", "name": "myvol", "schema_name": "myschema", + "volume_path": "/Volumes/mycat/myschema/myvol", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index c249c7b850..712c539cd0 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -232,6 +232,7 @@ "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", "updated_at": [UNIX_TIME_MILLIS][2], "volume_id": "[UUID]", + "volume_path": "/Volumes/mycat/myschema/myvol", "volume_type": "MANAGED" }, "changes": { diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 8a0116f88a..660aca1185 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -75,6 +75,7 @@ "catalog_name": "main", "name": "volume_name", "schema_name": "schema_grants", + "volume_path": "/Volumes/main/schema_grants/volume_name", "volume_type": "MANAGED" }, "depends_on": [ diff --git a/acceptance/bundle/python/volumes-support/output.txt b/acceptance/bundle/python/volumes-support/output.txt index e2b6d8771e..eafa1e8685 100644 --- a/acceptance/bundle/python/volumes-support/output.txt +++ b/acceptance/bundle/python/volumes-support/output.txt @@ -17,12 +17,14 @@ "catalog_name": "my_catalog", "name": "My Volume (updated)", "schema_name": "my_schema", + "volume_path": "/Volumes/my_catalog/my_schema/My Volume (updated)", "volume_type": "MANAGED" }, "my_volume_2": { "catalog_name": "my_catalog_2", "name": "My Volume (2) (updated)", "schema_name": "my_schema_2", + "volume_path": "/Volumes/my_catalog_2/my_schema_2/My Volume (2) (updated)", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 3ebaeaa628..e44e4100f6 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2975,6 +2975,7 @@ resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_path string ALL resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL resources.volumes.*.grants.grants []dresources.GrantAssignment ALL diff --git a/acceptance/bundle/resource_deps/bad_syntax/output.txt b/acceptance/bundle/resource_deps/bad_syntax/output.txt index 0e9d83b643..f03ea5f40d 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/output.txt +++ b/acceptance/bundle/resource_deps/bad_syntax/output.txt @@ -6,12 +6,14 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { "catalog_name": "${resources.volumes.bar.bad..syntax}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/${resources.volumes.bar.bad..syntax}/${resources.volumes.bar.schema_name}/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt index 841f33c1d2..08f77e1611 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt @@ -1,5 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in dresources.VolumeRemote Error: planning failed diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt b/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt index 6a2b5d1157..cef21f8017 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt @@ -1,4 +1,4 @@ -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in dresources.VolumeRemote Error: planning failed diff --git a/acceptance/bundle/resource_deps/non_existent_field/output.txt b/acceptance/bundle/resource_deps/non_existent_field/output.txt index ec7de84868..300d26c066 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/output.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/output.txt @@ -6,12 +6,14 @@ "catalog_name": "mycatalog", "name": "barname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/barname", "volume_type": "MANAGED" }, "foo": { "catalog_name": "${resources.volumes.bar.non_existent}", "name": "myname", "schema_name": "${resources.volumes.bar.schema_name}", + "volume_path": "/Volumes/${resources.volumes.bar.non_existent}/${resources.volumes.bar.schema_name}/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt index ce5fcfb66b..60455aa651 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/output.txt @@ -12,6 +12,7 @@ "catalog_name": "main", "name": "volumebar-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/${resources.schemas.my.name}/volumebar-[UNIQUE_NAME]", "volume_type": "MANAGED" }, "foo": { @@ -19,6 +20,7 @@ "comment": "${resources.volumes.bar.storage_location}", "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/${resources.schemas.my.name}/volumefoo-[UNIQUE_NAME]", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl b/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl new file mode 100644 index 0000000000..ec1e261984 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +resources: + schemas: + my: + catalog_name: main + name: myschema-${UNIQUE_NAME} + + volumes: + bar: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumebar-${UNIQUE_NAME} + + foo: + catalog_name: main + schema_name: myschema-${UNIQUE_NAME} + name: volumefoo-${UNIQUE_NAME} + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.direct.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.direct.txt new file mode 100644 index 0000000000..5a4539339c --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.direct.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.direct.json new file mode 100644 index 0000000000..88d5e8dfe3 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.direct.json @@ -0,0 +1,29 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.terraform.json b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.terraform.json new file mode 100644 index 0000000000..1596f4b825 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.requests.terraform.json @@ -0,0 +1,41 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "/Volumes/main/myschema-[UNIQUE_NAME]/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "myschema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumefoo-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.terraform.txt new file mode 100644 index 0000000000..5a4539339c --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.deploy.terraform.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.direct.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.direct.txt new file mode 100644 index 0000000000..b4d427d485 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.direct.txt @@ -0,0 +1,19 @@ +The following resources will be deleted: + delete resources.schemas.my + delete resources.volumes.bar + delete resources.volumes.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.terraform.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.terraform.txt new file mode 100644 index 0000000000..b4d427d485 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy.terraform.txt @@ -0,0 +1,19 @@ +The following resources will be deleted: + delete resources.schemas.my + delete resources.volumes.bar + delete resources.volumes.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.bar + delete resources.volumes.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.direct.json b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.direct.json new file mode 100644 index 0000000000..ba286e1222 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.direct.json @@ -0,0 +1,15 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumefoo-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.terraform.json b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.terraform.json new file mode 100644 index 0000000000..ba286e1222 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.destroy_requests.terraform.json @@ -0,0 +1,15 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.myschema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumebar-[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.myschema-[UNIQUE_NAME].volumefoo-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.direct.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.direct.txt new file mode 100644 index 0000000000..80d724bd01 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.direct.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.terraform.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.terraform.txt new file mode 100644 index 0000000000..80d724bd01 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.plan_after_deploy.terraform.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml b/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml new file mode 100644 index 0000000000..1819a94c46 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/out.test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[CloudEnvs] + azure = false + gcp = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt b/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt new file mode 100644 index 0000000000..26c0ec5436 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/output.txt @@ -0,0 +1,36 @@ + +>>> [CLI] bundle validate -o json +{ + "schemas": { + "my": { + "catalog_name": "main", + "name": "myschema-[UNIQUE_NAME]" + } + }, + "volumes": { + "bar": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/${resources.schemas.my.name}/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "main", + "comment": "${resources.volumes.bar.volume_path}", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "${resources.schemas.my.name}", + "volume_path": "/Volumes/main/${resources.schemas.my.name}/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } +} + +>>> [CLI] bundle plan +create schemas.my +create volumes.bar +create volumes.foo + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged + +>>> print_requests.py --get //unity diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/script b/acceptance/bundle/resource_deps/remote_field_volume_path/script new file mode 100644 index 0000000000..0a798dc632 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/script @@ -0,0 +1,15 @@ +cleanup() { + $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt + print_requests.py --sort //unity &> out.destroy_requests.$DATABRICKS_BUNDLE_ENGINE.json +} + +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle plan +trace print_requests.py --get //unity +trap cleanup EXIT +trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --get //unity &> out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json + +# Both engines should show no drift after deploy: +$CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path/test.toml b/acceptance/bundle/resource_deps/remote_field_volume_path/test.toml new file mode 100644 index 0000000000..ce7d04e3af --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +# Each cloud has its specific storage url, we record AWS's one in the output (which is also emulated by Local): +CloudEnvs.gcp = false +CloudEnvs.azure = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/databricks.yml.tmpl b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/databricks.yml.tmpl new file mode 100644 index 0000000000..6c3ad34cac --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: testbundle-${UNIQUE_NAME} + +resources: + volumes: + bar: + catalog_name: main + schema_name: default + name: volumebar-${UNIQUE_NAME} + + foo: + catalog_name: main + schema_name: default + name: volumefoo-${UNIQUE_NAME} + comment: ${resources.volumes.bar.volume_path} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.plan.direct.json b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.plan.direct.json new file mode 100644 index 0000000000..bcc3e2d672 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.plan.direct.json @@ -0,0 +1,37 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.volumes.bar": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } + }, + "resources.volumes.foo": { + "depends_on": [ + { + "node": "resources.volumes.bar", + "label": "${resources.volumes.bar.volume_path}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "/Volumes/main/default/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.test.toml b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/output.txt b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/output.txt new file mode 100644 index 0000000000..26fea91f80 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/output.txt @@ -0,0 +1,50 @@ + +>>> [CLI] bundle validate -o json +{ + "bar": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + }, + "foo": { + "catalog_name": "main", + "comment": "${resources.volumes.bar.volume_path}", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> jq .plan."resources.volumes.bar" | {action, new_state: .new_state.value} out.plan.direct.json +{ + "action": "create", + "new_state": { + "catalog_name": "main", + "name": "volumebar-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumebar-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> jq .plan."resources.volumes.foo" | {action, depends_on, new_state: .new_state.value} out.plan.direct.json +{ + "action": "create", + "depends_on": [ + { + "node": "resources.volumes.bar", + "label": "${resources.volumes.bar.volume_path}" + } + ], + "new_state": { + "catalog_name": "main", + "comment": "/Volumes/main/default/volumebar-[UNIQUE_NAME]", + "name": "volumefoo-[UNIQUE_NAME]", + "schema_name": "default", + "volume_path": "/Volumes/main/default/volumefoo-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/script b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/script new file mode 100755 index 0000000000..407ca6d386 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/script @@ -0,0 +1,6 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq .resources.volumes +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace jq '.plan."resources.volumes.bar" | {action, new_state: .new_state.value}' out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace jq '.plan."resources.volumes.foo" | {action, depends_on, new_state: .new_state.value}' out.plan.$DATABRICKS_BUNDLE_ENGINE.json +rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/remote_field_volume_path_plan/test.toml b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resource_deps/remote_field_volume_path_plan/test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index 2f9e4d51a1..f7a5dc9f1b 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -24,6 +24,7 @@ "catalog_name": "main", "name": "volume_name", "schema_name": "schema_grants_[UNIQUE_NAME]", + "volume_path": "/Volumes/main/schema_grants_[UNIQUE_NAME]/volume_name", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index a6709a074f..2d9062eb21 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -34,6 +34,7 @@ "owner": "[USERNAME]", "schema_name": "schema_grants_[UNIQUE_NAME]", "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "volume_path": "/Volumes/main/schema_grants_[UNIQUE_NAME]/volume_name", "volume_type": "MANAGED" }, "changes": { diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index e68c981548..1fc4049796 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -7,6 +7,7 @@ "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -65,6 +66,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?o=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } @@ -116,5 +118,6 @@ Error: Resource catalog.VolumeInfo not found: main.myschema.myvolume "modified_status": "created", "name": "myvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index f6bfbfbc03..3821d1e1ba 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -12,6 +12,7 @@ "comment": "COMMENT1", "name": "mynewvolume", "schema_name": "myschema", + "volume_path": "/Volumes/main/myschema/mynewvolume", "volume_type": "MANAGED" } }, @@ -27,6 +28,7 @@ "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "volume_id": "[UUID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" }, "changes": { @@ -41,6 +43,12 @@ "action": "skip", "reason": "backend_default", "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + }, + "volume_path": { + "action": "update", + "old": "/Volumes/main/myschema/myvolume", + "new": "/Volumes/main/myschema/mynewvolume", + "remote": "/Volumes/main/myschema/myvolume" } } } diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index 1f1b78cab8..8d8b41b518 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?o=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/change-schema-name/output.txt b/acceptance/bundle/resources/volumes/change-schema-name/output.txt index 4d5145da9e..29f9ec1dbf 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-schema-name/output.txt @@ -28,6 +28,7 @@ Deployment complete! "name": "myvolume", "schema_name": "myschema", "url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?o=[NUMID]", + "volume_path": "/Volumes/main/myschema/myvolume", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index 4f44f4f642..121eaa6adb 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -9,6 +9,7 @@ "catalog_name": "mycatalog", "name": "myname", "schema_name": "myschema", + "volume_path": "/Volumes/mycatalog/myschema/myname", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 9fb8cf7ee3..8dc8f5daf5 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -35,6 +35,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -76,6 +77,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } @@ -117,6 +119,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json index cf1745cc2f..258b6b938a 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_empty_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json index cf1745cc2f..258b6b938a 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_nil_prefix.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json index eff500341a..3fe9aeea34 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.with_static_prefix.json @@ -57,6 +57,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json index cf1745cc2f..258b6b938a 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.without_presets.json @@ -63,6 +63,7 @@ "catalog_name": "catalog1", "name": "volume1", "schema_name": "schema1", + "volume_path": "/Volumes/catalog1/schema1/volume1", "volume_type": "MANAGED" } } diff --git a/bundle/config/mutator/python/apply_python_output.go b/bundle/config/mutator/python/apply_python_output.go index fb5452d3b4..875788eec3 100644 --- a/bundle/config/mutator/python/apply_python_output.go +++ b/bundle/config/mutator/python/apply_python_output.go @@ -2,11 +2,14 @@ package python import ( "fmt" + "reflect" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/structs/structpath" ) // applyPythonOutputResult contains which resources where added, updated, or deleted by Python mutator. @@ -25,7 +28,12 @@ type applyPythonOutputResult struct { // - if property is unchanged in output, it's original location will be preserved // - if empty sequence/mapping is deleted in output, it's original value will be preserved func applyPythonOutput(root, output dyn.Value) (dyn.Value, applyPythonOutputResult, error) { - result, visitor := createOverrideVisitor(root, output) + internalPatterns, err := internalFieldPatterns(reflect.TypeFor[config.Root]()) + if err != nil { + return dyn.InvalidValue, applyPythonOutputResult{}, fmt.Errorf("failed to find internal fields: %w", err) + } + + result, visitor := createOverrideVisitor(root, output, internalPatterns) merged, err := merge.Override(root, output, visitor) if err != nil { return dyn.InvalidValue, result, err @@ -34,14 +42,22 @@ func applyPythonOutput(root, output dyn.Value) (dyn.Value, applyPythonOutputResu return merged, result, nil } -func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResult, merge.OverrideVisitor) { +func createOverrideVisitor(leftRoot, rightRoot dyn.Value, internalPatterns []*structpath.PatternNode) (applyPythonOutputResult, merge.OverrideVisitor) { resourcesPath := dyn.NewPath(dyn.Key("resources")) deleted := resourcemutator.NewResourceKeySet() updated := resourcemutator.NewResourceKeySet() added := resourcemutator.NewResourceKeySet() + isInternalPath := func(valuePath dyn.Path) bool { + return matchesAnyPattern(dynPathToStructPath(valuePath), internalPatterns) + } + visitor := merge.OverrideVisitor{ VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { + if isInternalPath(valuePath) { + return merge.ErrOverrideUndoDelete + } + if isOmitemptyDelete(left) { return merge.ErrOverrideUndoDelete } @@ -89,6 +105,10 @@ func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResu } }, VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { + if isInternalPath(valuePath) { + return right, nil + } + if !valuePath.HasPrefix(resourcesPath) { return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String()) } @@ -135,7 +155,11 @@ func createOverrideVisitor(leftRoot, rightRoot dyn.Value) (applyPythonOutputResu return right, nil } }, - VisitUpdate: func(valuePath dyn.Path, _, right dyn.Value) (dyn.Value, error) { + VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { + if isInternalPath(valuePath) { + return left, nil + } + if !valuePath.HasPrefix(resourcesPath) { return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String()) } diff --git a/bundle/config/mutator/python/apply_python_output_test.go b/bundle/config/mutator/python/apply_python_output_test.go index 2d64d6c4bb..b6e91b07c4 100644 --- a/bundle/config/mutator/python/apply_python_output_test.go +++ b/bundle/config/mutator/python/apply_python_output_test.go @@ -157,6 +157,29 @@ func TestApplyPythonOutput(t *testing.T) { } } +func TestApplyPythonOutput_ignoresInternalFields(t *testing.T) { + input := mapOf("resources", mapOf("volumes", mapOf("volume_1", mapOf2( + "name", dyn.V("volume"), + "volume_path", dyn.V("/Volumes/main/default/volume"), + )))) + output := mapOf("resources", mapOf("volumes", mapOf("volume_1", mapOf( + "name", dyn.V("volume"), + )))) + + merged, state, err := applyPythonOutput(input, output) + + assert.NoError(t, err) + assert.Empty(t, state.UpdatedResources.ToArray()) + + name, err := dyn.GetByPath(merged, dyn.MustPathFromString("resources.volumes.volume_1.name")) + assert.NoError(t, err) + assert.Equal(t, "volume", name.MustString()) + + volumePath, err := dyn.GetByPath(merged, dyn.MustPathFromString("resources.volumes.volume_1.volume_path")) + assert.NoError(t, err) + assert.Equal(t, "/Volumes/main/default/volume", volumePath.MustString()) +} + func TestMergeOutput_disallowDelete(t *testing.T) { input := mapOf("not_resource", dyn.V("value")) output := emptyMap() @@ -244,7 +267,7 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, visitor := createOverrideVisitor(dyn.NilValue, dyn.NilValue) + _, visitor := createOverrideVisitor(dyn.NilValue, dyn.NilValue, nil) err := visitor.VisitDelete(tc.path, tc.left) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index c20e172c00..826aedf4a1 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -32,6 +32,9 @@ import ( "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structwalk" ) type phase string @@ -436,6 +439,11 @@ func explainProcessErr(stderr string) string { } func writeInputFile(inputPath string, input dyn.Value) error { + input, err := stripInternalFields(input, reflect.TypeFor[config.Root]()) + if err != nil { + return fmt.Errorf("failed to strip internal fields: %w", err) + } + // we need to marshal dyn.Value instead of bundle.Config to JSON to support // non-string fields assigned with bundle variables rootConfigJson, err := json.Marshal(input.AsAny()) @@ -446,6 +454,73 @@ func writeInputFile(inputPath string, input dyn.Value) error { return os.WriteFile(inputPath, rootConfigJson, 0o600) } +func stripInternalFields(input dyn.Value, typ reflect.Type) (dyn.Value, error) { + patterns, err := internalFieldPatterns(typ) + if err != nil { + return dyn.InvalidValue, err + } + + if len(patterns) == 0 { + return input, nil + } + + return dyn.Walk(input, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + if matchesAnyPattern(dynPathToStructPath(path), patterns) { + return dyn.InvalidValue, dyn.ErrDrop + } + + return value, nil + }) +} + +func internalFieldPatterns(typ reflect.Type) ([]*structpath.PatternNode, error) { + var patterns []*structpath.PatternNode + + err := structwalk.WalkType(typ, func(path *structpath.PatternNode, _ reflect.Type, field *reflect.StructField) bool { + if path.IsRoot() || field == nil { + return true + } + + if structtag.BundleTag(field.Tag.Get("bundle")).Internal() { + patterns = append(patterns, path) + return false + } + + return true + }) + + return patterns, err +} + +func dynPathToStructPath(path dyn.Path) *structpath.PathNode { + var out *structpath.PathNode + + for _, component := range path { + if key := component.Key(); key != "" { + out = structpath.NewStringKey(out, key) + continue + } + + out = structpath.NewIndex(out, component.Index()) + } + + return out +} + +func matchesAnyPattern(path *structpath.PathNode, patterns []*structpath.PatternNode) bool { + if path == nil { + return false + } + + for _, pattern := range patterns { + if path.Len() == pattern.Len() && path.HasPatternPrefix(pattern) { + return true + } + } + + return false +} + // loadLocationsFile loads locations.json containing source locations for generated YAML. func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, error) { locationsFile, err := os.Open(locationsPath) diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 285d1b3b87..f08d1678fd 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -6,9 +6,13 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "testing" + "github.com/databricks/databricks-sdk-go/service/catalog" + + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/bundle/env" @@ -287,6 +291,34 @@ func TestPythonMutator_disabled(t *testing.T) { assert.NoError(t, diag.Error()) } +func TestStripInternalFields(t *testing.T) { + input, err := convert.FromTyped(config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "default", + Name: "my_volume", + }, + VolumePath: "/Volumes/main/default/my_volume", + }, + }, + }, + }, dyn.NilValue) + require.NoError(t, err) + + output, err := stripInternalFields(input, reflect.TypeFor[config.Root]()) + require.NoError(t, err) + + _, err = dyn.GetByPath(output, dyn.MustPathFromString("resources.volumes.foo.volume_path")) + require.Error(t, err) + + name, err := dyn.GetByPath(output, dyn.MustPathFromString("resources.volumes.foo.name")) + require.NoError(t, err) + assert.Equal(t, "my_volume", name.MustString()) +} + func TestPythonMutator_venvNotFound(t *testing.T) { expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path")) diff --git a/bundle/config/mutator/resourcemutator/capture_schema_dependency.go b/bundle/config/mutator/resourcemutator/capture_schema_dependency.go index ef1581393a..77434a6ae9 100644 --- a/bundle/config/mutator/resourcemutator/capture_schema_dependency.go +++ b/bundle/config/mutator/resourcemutator/capture_schema_dependency.go @@ -37,6 +37,14 @@ func catalogNameRef(key string) string { return fmt.Sprintf("${resources.catalogs.%s.name}", key) } +func volumePath(catalogName, schemaName, name string) string { + if catalogName == "" || schemaName == "" || name == "" { + return "" + } + + return fmt.Sprintf("/Volumes/%s/%s/%s", catalogName, schemaName, name) +} + func findSchema(b *bundle.Bundle, catalogName, schemaName string) (string, *resources.Schema) { if catalogName == "" || schemaName == "" { return "", nil @@ -55,11 +63,11 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) { return } schemaK, schema := findSchema(b, v.CatalogName, v.SchemaName) - if schema == nil { - return + if schema != nil { + v.SchemaName = schemaNameRef(schemaK) } - v.SchemaName = schemaNameRef(schemaK) + v.VolumePath = volumePath(v.CatalogName, v.SchemaName, v.Name) } func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { diff --git a/bundle/config/mutator/resourcemutator/capture_schema_dependency_test.go b/bundle/config/mutator/resourcemutator/capture_schema_dependency_test.go index 8e37a9ba16..868618b486 100644 --- a/bundle/config/mutator/resourcemutator/capture_schema_dependency_test.go +++ b/bundle/config/mutator/resourcemutator/capture_schema_dependency_test.go @@ -44,30 +44,35 @@ func TestCaptureSchemaDependencyForVolume(t *testing.T) { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "catalog1", SchemaName: "foobar", + Name: "volume1", }, }, "volume2": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "catalog2", SchemaName: "foobar", + Name: "volume2", }, }, "volume3": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "catalog1", SchemaName: "barfoo", + Name: "volume3", }, }, "volume4": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "catalogX", SchemaName: "foobar", + Name: "volume4", }, }, "volume5": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ CatalogName: "catalog1", SchemaName: "schemaX", + Name: "volume5", }, }, "nilVolume": nil, @@ -87,7 +92,12 @@ func TestCaptureSchemaDependencyForVolume(t *testing.T) { assert.Equal(t, "schemaX", b.Config.Resources.Volumes["volume5"].SchemaName) assert.Nil(t, b.Config.Resources.Volumes["nilVolume"]) - // assert.Nil(t, b.Config.Resources.Volumes["emptyVolume"].CreateVolumeRequestContent) + assert.Equal(t, "/Volumes/catalog1/${resources.schemas.schema1.name}/volume1", b.Config.Resources.Volumes["volume1"].VolumePath) + assert.Equal(t, "/Volumes/catalog2/${resources.schemas.schema2.name}/volume2", b.Config.Resources.Volumes["volume2"].VolumePath) + assert.Equal(t, "/Volumes/catalog1/${resources.schemas.schema3.name}/volume3", b.Config.Resources.Volumes["volume3"].VolumePath) + assert.Equal(t, "/Volumes/catalogX/foobar/volume4", b.Config.Resources.Volumes["volume4"].VolumePath) + assert.Equal(t, "/Volumes/catalog1/schemaX/volume5", b.Config.Resources.Volumes["volume5"].VolumePath) + assert.Equal(t, "", b.Config.Resources.Volumes["emptyVolume"].VolumePath) } func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { diff --git a/bundle/config/resources/volume.go b/bundle/config/resources/volume.go index 8c47a6afc4..4c47ff79ba 100644 --- a/bundle/config/resources/volume.go +++ b/bundle/config/resources/volume.go @@ -44,6 +44,8 @@ type Volume struct { BaseResource catalog.CreateVolumeRequestContent + VolumePath string `json:"volume_path,omitempty" bundle:"internal"` + // List of grants to apply on this volume. Grants []VolumeGrant `json:"grants,omitempty"` } diff --git a/bundle/deploy/terraform/tfdyn/convert_volume.go b/bundle/deploy/terraform/tfdyn/convert_volume.go index 287ddee0c6..e9cb807fa7 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume.go @@ -11,6 +11,12 @@ import ( ) func convertVolumeResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + var err error + vin, err = dyn.Set(vin, "volume_path", dyn.InvalidValue) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourceVolume{}, vin) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_volume_test.go b/bundle/deploy/terraform/tfdyn/convert_volume_test.go index 92c64212b9..dd31dd1256 100644 --- a/bundle/deploy/terraform/tfdyn/convert_volume_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_volume_test.go @@ -23,6 +23,7 @@ func TestConvertVolume(t *testing.T) { StorageLocation: "s3://bucket/path", VolumeType: "EXTERNAL", }, + VolumePath: "/Volumes/catalog/schema/name", Grants: []resources.VolumeGrant{ { Privileges: []resources.VolumeGrantPrivilege{ diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 6d33a49e27..c7586be6e8 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -9,9 +9,79 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +const volumePathPrefix = "/Volumes/" + +type VolumeState struct { + catalog.CreateVolumeRequestContent + + VolumePath string `json:"volume_path,omitempty"` +} + +// UnmarshalJSON preserves extra fields alongside the embedded SDK type. +func (s *VolumeState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +// MarshalJSON preserves extra fields alongside the embedded SDK type. +func (s VolumeState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type VolumeRemote struct { + catalog.VolumeInfo + + VolumePath string `json:"volume_path,omitempty"` +} + +// Custom marshaler needed because embedded VolumeInfo has its own MarshalJSON +// which would otherwise take over and ignore additional fields. +func (s *VolumeRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VolumeRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func makeVolumePath(catalogName, schemaName, name string) string { + if catalogName == "" || schemaName == "" || name == "" { + return "" + } + + return volumePathPrefix + catalogName + "/" + schemaName + "/" + name +} + +func makeVolumeState(input *resources.Volume) *VolumeState { + if input == nil { + return nil + } + + volumePath := input.VolumePath + if volumePath == "" { + volumePath = makeVolumePath(input.CatalogName, input.SchemaName, input.Name) + } + + return &VolumeState{ + CreateVolumeRequestContent: input.CreateVolumeRequestContent, + VolumePath: volumePath, + } +} + +func makeVolumeRemote(info *catalog.VolumeInfo) *VolumeRemote { + if info == nil { + return nil + } + + return &VolumeRemote{ + VolumeInfo: *info, + VolumePath: makeVolumePath(info.CatalogName, info.SchemaName, info.Name), + } +} + type ResourceVolume struct { client *databricks.WorkspaceClient } @@ -20,35 +90,43 @@ func (*ResourceVolume) New(client *databricks.WorkspaceClient) *ResourceVolume { return &ResourceVolume{client: client} } -func (*ResourceVolume) PrepareState(input *resources.Volume) *catalog.CreateVolumeRequestContent { - return &input.CreateVolumeRequestContent +func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { + return makeVolumeState(input) } -func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolumeRequestContent { - return &catalog.CreateVolumeRequestContent{ - CatalogName: info.CatalogName, - Comment: info.Comment, - Name: info.Name, - SchemaName: info.SchemaName, - StorageLocation: info.StorageLocation, - VolumeType: info.VolumeType, - ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), +func (*ResourceVolume) RemapState(info *VolumeRemote) *VolumeState { + state := &VolumeState{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: info.CatalogName, + Comment: info.Comment, + Name: info.Name, + SchemaName: info.SchemaName, + StorageLocation: info.StorageLocation, + VolumeType: info.VolumeType, + ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), + }, + VolumePath: info.VolumePath, } + return state } -func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*catalog.VolumeInfo, error) { - return r.client.Volumes.ReadByName(ctx, id) +func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*VolumeRemote, error) { + response, err := r.client.Volumes.ReadByName(ctx, id) + if err != nil { + return nil, err + } + return makeVolumeRemote(response), nil } -func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) { - response, err := r.client.Volumes.Create(ctx, *config) +func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (string, *VolumeRemote, error) { + response, err := r.client.Volumes.Create(ctx, config.CreateVolumeRequestContent) if err != nil { return "", nil, err } - return response.FullName, response, nil + return response.FullName, makeVolumeRemote(response), nil } -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ Changes) (*catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ Changes) (*VolumeRemote, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -76,10 +154,10 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo log.Warnf(ctx, "volumes: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } - return response, err + return makeVolumeRemote(response), err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState) (string, *VolumeRemote, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -105,7 +183,7 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return "", nil, err } - return response.FullName, response, nil + return response.FullName, makeVolumeRemote(response), nil } func (r *ResourceVolume) DoDelete(ctx context.Context, id string) error { diff --git a/bundle/direct/dresources/volume_test.go b/bundle/direct/dresources/volume_test.go new file mode 100644 index 0000000000..04c3752640 --- /dev/null +++ b/bundle/direct/dresources/volume_test.go @@ -0,0 +1,78 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" +) + +func TestResourceVolumePrepareState(t *testing.T) { + r := &ResourceVolume{} + + state := r.PrepareState(&resources.Volume{ + CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvolume", + Comment: "comment", + VolumeType: catalog.VolumeTypeManaged, + }, + }) + + assert.Equal(t, "main", state.CatalogName) + assert.Equal(t, "myschema", state.SchemaName) + assert.Equal(t, "myvolume", state.Name) + assert.Equal(t, "/Volumes/main/myschema/myvolume", state.VolumePath) +} + +func TestMakeVolumeRemote_FromFullName(t *testing.T) { + info := &catalog.VolumeInfo{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvolume", + FullName: "main.myschema.myvolume", + } + + remote := makeVolumeRemote(info) + + assert.Equal(t, "/Volumes/main/myschema/myvolume", remote.VolumePath) + assert.Equal(t, "main.myschema.myvolume", remote.FullName) +} + +func TestMakeVolumeRemote_FallbackFromNameParts(t *testing.T) { + info := &catalog.VolumeInfo{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvolume", + } + + remote := makeVolumeRemote(info) + + assert.Equal(t, "/Volumes/main/myschema/myvolume", remote.VolumePath) +} + +func TestResourceVolumeRemapState(t *testing.T) { + r := &ResourceVolume{} + + state := r.RemapState(&VolumeRemote{ + VolumeInfo: catalog.VolumeInfo{ + CatalogName: "main", + SchemaName: "myschema", + Name: "myvolume", + Comment: "comment", + StorageLocation: "s3://bucket/path", + VolumeType: catalog.VolumeTypeManaged, + }, + VolumePath: "/Volumes/main/myschema/myvolume", + }) + + assert.Equal(t, "main", state.CatalogName) + assert.Equal(t, "myschema", state.SchemaName) + assert.Equal(t, "myvolume", state.Name) + assert.Equal(t, "comment", state.Comment) + assert.Equal(t, "s3://bucket/path", state.StorageLocation) + assert.Equal(t, catalog.VolumeTypeManaged, state.VolumeType) + assert.Equal(t, "/Volumes/main/myschema/myvolume", state.VolumePath) +}