Support creation of CDP DW Database Catalog and Virtual Warehouses

roles/common/defaults/main.yml

@@ -132,6 +132,9 @@ common__setup_runtime:                    "{{ ml is defined or de is defined or
 common__setup_plat:                       "{{ env is defined or sequence__setup_runtime  | default(False) | bool }}"
 common__setup_infra:                      "{{ infra is defined or sequence__setup_plat | default(False) | bool }}"
 common__setup_base:                       "{{ mgmt is defined or clusters is defined | default(False) | bool }}"
+common__teardown_runtime:                 "{{ common__setup_runtime }}"
+common__teardown_plat:                    "{{ common__setup_plat and dw.teardown is undefined }}"
+common__teardown_infra:                   "{{ common__setup_infra and dw.teardown is undefined }}"
 
 common__include_ml:                        "{{ ml is defined | bool }}"
 common__include_dw:                        "{{ dw is defined | bool }}"

roles/runtime/defaults/main.yml

@@ -65,8 +65,17 @@ run__ml_remove_storage:             "{{ ml.remove_storage | default (run__force_
 run__ml_public_loadbalancer:        "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}"
 
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
-run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
+run__dw_dbc_suffix:                 "{{ dw.dbc.suffix | default('dbc') }}"
+run__dw_vw_suffix:                  "{{ dw.vw.suffix | default('vw') }}"
+run__dw_tags:                       "{{ dw.tags | default(common__tags) }}"
+run__dw_overlay_network:            "{{ dw.overlay_network | bool }}"
+run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
+run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
+run__dw_default_vw_type:            "{{ dw.default_vw_type | default('hive') }}"
+run__dw_default_template_type:      "{{ dw.default_template_type | default('xsmall') }}"
+run__dw_default_dbc_suffix:         "{{ dw.default_dbc_suffix | default('-dl-default') }}"
+run__dw_teardown:                   "{{ dw.teardown | default(omit) }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"

roles/runtime/tasks/initialize_base.yml

@@ -161,3 +161,44 @@
       loop_control:
         loop_var: __ml_config
         label: "{{ config.name }}"
+
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
+  block:
+    - name: Construct CDP DW Data catalog configurations
+      ansible.builtin.set_fact:
+        run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        config:
+          name: "{{ __dw_config.name | default('') }}"
+          load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
+          use_default_dbc: "{{ __dw_config.name is undefined | bool }}"
+          virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
+      loop: "{{ run__dw_definitions }}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_config_index
+
+    - name: Construct CDP DW Virtual warehouse configurations
+      ansible.builtin.set_fact:
+        run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
+      vars:
+        config:
+          name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          dbc_name: "{{ __dw_config.0.name }}"
+          use_default_dbc: "{{ __dw_config.0.use_default_dbc }}"
+          type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}"
+          template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}"
+          autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}"
+          autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}"
+          common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}"
+          application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}"
+          ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}"
+          enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}"
+          tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}"
+      loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_dbc_index
+        label: "{{ __dw_config.0.name }}"

roles/runtime/tasks/initialize_setup_aws.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 - name: Discover AWS Public and Private VPC Subnets
-  when: not infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is undefined
   block:
     - name: Query AWS Public Subnets
       amazon.aws.ec2_vpc_subnet_info:
@@ -49,13 +49,17 @@
         label: "{{ __aws_private_subnet_item.subnet_id }}"
       loop: "{{ __aws_private_subnets_info.subnets }}"
 
+    - name: Set default AWS Private Subnet if there are no Private Subnets
+      ansible.builtin.set_fact:
+        run__datahub_private_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) }}"
+
     - name: Set fact for AWS Subnet IDs
       when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined
       ansible.builtin.set_fact:
-        run__datahub_subnet_ids: "{{ run__datahub_public_subnet_ids | default([]) | union(run__datahub_private_subnet_ids) }}"
+        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | union(run__datahub_public_subnet_ids) }}"
 
 - name: Set fact for AWS Subnet IDs by assignment
-  when: infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is defined
   block:
     - name: Set fact for All AWS Subnet IDs by assignment
       ansible.builtin.set_fact:

roles/runtime/tasks/initialize_teardown.yml

@@ -68,4 +68,111 @@
       loop: "{{ run__env_info.environments[0].descendants.ml }}"
       loop_control:
         loop_var: __ml_config
-        label: "{{ __ml_config.instanceName }}"
+        label: "{{ __ml_config.instanceName }}"
+
+- name: Discover CDW deployments
+  when: run__include_dw
+  block:
+    - name: Retrieve CDP DW experiences
+      cloudera.cloud.dw_cluster_info:
+        env: "{{ run__env_name }}"
+      register: __dw_list
+
+    - name: Initialize CDP DW cluster id
+      ansible.builtin.set_fact:
+        run__dw_cluster_id: "{{ __dw_list.clusters[0].id }}"
+
+    - name: Prepare partial teardown for CDW resources
+      when: run__dw_teardown is defined
+      block:
+        - name: Initialize list of CDW database catalogs to be decommissioned
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ [] }}"
+
+        - name: Initialize list of CDW virtual warehouses to be decommissioned
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ [] }}"
+
+        - name: Populate CDW database catalogs defined with its id to the teardown list
+          when: __dbc.id is defined
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc.id ] }}"
+          loop: "{{ run__dw_teardown.database_catalogs }}"
+          loop_control:
+            loop_var: __dbc
+            label: "{{ __dbc.id }}"
+
+        - name: Fetch the ids of all the CDW database catalogs defined with its name
+          when: __dbc.name is defined
+          cloudera.cloud.dw_database_catalog_info:
+             name: "{{ __dbc.name }}"
+             cluster_id: "{{ run__dw_cluster_id }}"
+          loop: "{{ run__dw_teardown.database_catalogs }}"
+          loop_control:
+            loop_var: __dbc
+            label: "{{ __dbc.name }}"
+          register: __dbc_name_list
+
+        - name: Populate CDW database catalogs defined with its name to the teardown list
+          when:
+              - __dbc_result.database_catalogs is defined
+              - __dbc_result.database_catalogs | length > 0
+              - __dbc_result.database_catalogs[0].id not in run__dw_dbc_ids
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc_result.database_catalogs[0].id ] }}"
+          loop: "{{ __dbc_name_list.results }}"
+          loop_control:
+            loop_var: __dbc_result
+            label: "{{ __dbc_result.database_catalogs[0].id }}"
+
+        - name: Fetch all the CDW virtual warehouses under a CDW cluster
+          cloudera.cloud.dw_virtual_warehouse_info:
+            cluster_id: "{{ run__dw_cluster_id }}"
+          register: __all_vw_list
+
+        - name: Populate all the CDW virtual warehouses of database catalogs scheduled for teardown to the teardown list
+          when:
+              - run__dw_dbc_ids is defined
+              - __vw.dbcId in run__dw_dbc_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ] }}"
+          loop: "{{ __all_vw_list.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.id }}"
+
+        - name: Fetch the ids of all the CDW virtual warehouses defined with its name
+          when:
+               - run__dw_teardown.virtual_warehouses is defined
+               - __vw.name is defined
+          cloudera.cloud.dw_virtual_warehouse_info:
+            name: "{{ __vw.name }}"
+            cluster_id: "{{ run__dw_cluster_id }}"
+          loop: "{{ run__dw_teardown.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.name }}"
+          register: __vw_name_list
+
+        - name: Populate CDW virtual warehouses defined with its id to the teardown list
+          when:
+               - __vw.id is defined
+               - __vw.id not in run__dw_vw_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ]}}"
+          loop: "{{ run__dw_teardown.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.id }}"
+
+        - name: Populate CDW virtual warehouses defined with its name to the teardown list
+          when:
+               - __vw_result.virtual_warehouses is defined
+               - __vw_result.virtual_warehouses | length > 0
+               - __vw_result.virtual_warehouses[0].id not in run__dw_vw_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw_result.virtual_warehouses[0].id ]}}"
+          loop: "{{ __vw_name_list.results }}"
+          loop_control:
+            loop_var: __vw_result
+            label: "{{ __vw_result.virtual_warehouses[0].id }}"

roles/runtime/tasks/setup_aws.yml

@@ -40,10 +40,10 @@
     - name: Execute CDP DW cluster setup
       cloudera.cloud.dw_cluster:
         env: "{{ run__env_name }}"
-        overlay: no
-        # TODO - Allow direct assignment (will need to coordinate with infra role)
+        overlay: "{{ run__dw_overlay_network }}"
+        private_load_balancer: "{{ run__dw_private_load_balancer }}"
         aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
-        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+        aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}"
         state: present
         wait: yes
       async: 3600 # 1 hour timeout
@@ -60,4 +60,95 @@
       register: __dw_builds_async
       until: __dw_builds_async.finished
       retries: 120
-      delay: 30
+      delay: 30
+
+    - name: Retrieve CDP DW experiences
+      cloudera.cloud.dw_cluster_info:
+        env: "{{ run__env_name }}"
+      register: run__dw_list
+
+    - name: Fetch all the Database Catalogs under the cluster
+      cloudera.cloud.dw_database_catalog_info:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+      register: __dbc_list
+
+    - name: Set default Database catalog id for the cluster
+      when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$')
+      ansible.builtin.set_fact:
+        __default_dbc_id: "{{ __dw_dbc.id }}"
+      loop: "{{ __dbc_list.database_catalogs }}"
+      loop_control:
+        loop_var: __dw_dbc
+
+    - name: Create CDP DW Database catalogs
+      when: not __dw_dbc_config.use_default_dbc
+      cloudera.cloud.dw_database_catalog:
+        cluster_id : "{{ run__dw_list.clusters[0].id }}"
+        name: "{{ __dw_dbc_config.name }}"
+        load_demo_data: "{{ __dw_dbc_config.load_demo_data }}"
+        state: present
+        wait: yes
+      async: 3600 # 1 hour timeout
+      poll: 0
+      loop: "{{ run__dw_dbc_configs }}"
+      loop_control:
+        loop_var: __dw_dbc_config
+        label: "{{ __dw_dbc_config.name }}"
+      register: __dw_dbc_builds
+
+    - name: Wait for CDP DW Database catalogs setup to complete
+      when: __dw_dbc_build.ansible_job_id is defined
+      ansible.builtin.async_status:
+        jid: "{{ __dw_dbc_build.ansible_job_id }}"
+      register: __dw_dbc_builds_async
+      until: __dw_dbc_builds_async.finished
+      retries: 120
+      delay: 30
+      loop: "{{ __dw_dbc_builds.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build
+        label: "{{ __dw_dbc_build.__dw_dbc_config.name }}"
+
+    - name: Set CDP DW Database catalog name to id map
+      when: __dw_dbc_build_async.database_catalog is defined
+      ansible.builtin.set_fact:
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}"
+      loop: "{{ __dw_dbc_builds_async.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build_async
+
+    - name: Create CDP DW Virtual warehouse
+      cloudera.cloud.dw_virtual_warehouse:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+        dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}"
+        type: "{{ __dw_vw_config.type }}"
+        name: "{{ __dw_vw_config.name }}"
+        template: "{{ __dw_vw_config.template }}"
+        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}"
+        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}"
+        common_configs: "{{ __dw_vw_config.common_configs }}"
+        application_configs: "{{ __dw_vw_config.application_configs }}"
+        ldap_groups: "{{ __dw_vw_config.ldap_groups }}"
+        enable_sso: "{{ __dw_vw_config.enable_sso | bool }}"
+        tags: "{{ __dw_vw_config.tags }}"
+        wait: yes
+      async: 3600 # 1 hour timeout
+      poll: 0
+      register: __dw_vw_builds
+      loop: "{{ run__dw_vw_configs }}"
+      loop_control:
+        loop_var: __dw_vw_config
+        label: "{{ __dw_vw_config.name }}"
+
+    - name: Wait for CDP DW Virtual warehouse setup to complete
+      when: __dw_vw_build.ansible_job_id is defined
+      ansible.builtin.async_status:
+        jid: "{{ __dw_vw_build.ansible_job_id }}"
+      register: __dw_vw_builds_async
+      until: __dw_vw_builds_async.finished
+      retries: 120
+      delay: 30
+      loop: "{{ __dw_vw_builds.results }}"
+      loop_control:
+        loop_var: __dw_vw_build
+        label: "{{ __dw_vw_build.__dw_vw_config.name }}"