Address review comments

Author: raju-saravanan

roles/runtime/defaults/main.yml

@@ -72,6 +72,9 @@ run__dw_overlay_network:            "{{ dw.overlay_network | bool }}"
 run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
 run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
+run__dw_default_vw_type:            "{{ dw.default_vw_type | default('hive') }}"
+run__dw_default_template_type:      "{{ dw.default_template_type | default('xsmall') }}"
+run__dw_default_dbc_suffix:         "{{ dw.default_dbc_suffix | default('-dl-default') }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"

roles/runtime/tasks/initialize_base.yml

@@ -170,11 +170,10 @@
         run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
       vars:
         include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
-        use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}"
         config:
           name: "{{ __dw_config.name | default('') }}"
           load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
-          use_default_dbc: "{{ use_default_dbc }}"
+          use_default_dbc: "{{ __dw_config.name is undefined | bool }}"
           virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
       loop: "{{ run__dw_definitions }}"
       loop_control:
@@ -186,19 +185,20 @@
         run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
       vars:
         config:
-          name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
-          dbc_name: "{{ item.0.name }}"
-          use_default_dbc: "{{ item.0.use_default_dbc }}"
-          type: "{{ item.1.type | default('hive') }}"
-          template: "{{ item.1.template | default('xsmall') }}"
-          autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}"
-          autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}"
-          common_configs: "{{ item.1.configs.common_configs | default({}) }}"
-          application_configs: "{{ item.1.configs.application_configs | default({}) }}"
-          ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}"
-          enable_sso: "{{ item.1.configs.enable_sso | default(None) }}"
-          tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}"
+          name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          dbc_name: "{{ __dw_config.0.name }}"
+          use_default_dbc: "{{ __dw_config.0.use_default_dbc }}"
+          type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}"
+          template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}"
+          autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}"
+          autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}"
+          common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}"
+          application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}"
+          ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}"
+          enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}"
+          tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}"
       loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
       loop_control:
+        loop_var: __dw_config
         index_var: __dw_dbc_index
-        label: "{{ item.0.name }}"
+        label: "{{ __dw_config.0.name }}"

roles/runtime/tasks/initialize_setup_aws.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 - name: Discover AWS Public and Private VPC Subnets
-  when: not infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is undefined
   block:
     - name: Query AWS Public Subnets
       amazon.aws.ec2_vpc_subnet_info:
@@ -49,18 +49,17 @@
         label: "{{ __aws_private_subnet_item.subnet_id }}"
       loop: "{{ __aws_private_subnets_info.subnets }}"
 
-    - name: Assign AWS Private Subnet IDs in not assinged
-      when: run__datahub_private_subnet_ids is undefined
+    - name: Set default AWS Private Subnet if there are no Private Subnets
       ansible.builtin.set_fact:
-        run__datahub_private_subnet_ids: "{{ [] }}"
+        run__datahub_private_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) }}"
 
     - name: Set fact for AWS Subnet IDs
       when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined
       ansible.builtin.set_fact:
-        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}"
+        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | union(run__datahub_public_subnet_ids) }}"
 
 - name: Set fact for AWS Subnet IDs by assignment
-  when: infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is defined
   block:
     - name: Set fact for All AWS Subnet IDs by assignment
       ansible.builtin.set_fact:

roles/runtime/tasks/setup_aws.yml

@@ -67,6 +67,19 @@
         env: "{{ run__env_name }}"
       register: run__dw_list
 
+    - name: Fetch all the Database Catalogs under the cluster
+      cloudera.cloud.dw_database_catalog_info:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+      register: __dbc_list
+
+    - name: Set default Database catalog id for the cluster
+      when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$')
+      ansible.builtin.set_fact:
+        __default_dbc_id: "{{ __dw_dbc.id }}"
+      loop: "{{ __dbc_list.database_catalogs }}"
+      loop_control:
+        loop_var: __dw_dbc
+
     - name: Create CDP DW Database catalogs
       when: not __dw_dbc_config.use_default_dbc
       cloudera.cloud.dw_database_catalog:
@@ -97,22 +110,22 @@
         label: "{{ __dw_dbc_build.__dw_dbc_config.name }}"
 
     - name: Set CDP DW Database catalog name to id map
-      when: __dw_dbc_build_async.database_catalogs is defined
+      when: __dw_dbc_build_async.database_catalog is defined
       ansible.builtin.set_fact:
-        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}"
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}"
       loop: "{{ __dw_dbc_builds_async.results }}"
       loop_control:
         loop_var: __dw_dbc_build_async
 
     - name: Create CDP DW Virtual warehouse
       cloudera.cloud.dw_virtual_warehouse:
         cluster_id: "{{ run__dw_list.clusters[0].id }}"
-        dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}"
+        dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}"
         type: "{{ __dw_vw_config.type }}"
         name: "{{ __dw_vw_config.name }}"
         template: "{{ __dw_vw_config.template }}"
-        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}"
-        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}"
+        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}"
+        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}"
         common_configs: "{{ __dw_vw_config.common_configs }}"
         application_configs: "{{ __dw_vw_config.application_configs }}"
         ldap_groups: "{{ __dw_vw_config.ldap_groups }}"