From 3d54286a21840981250e5e97b3d969c5547b0fd8 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Thu, 15 Jul 2021 12:02:28 +0530
Subject: [PATCH 01/10]   Support CDW VW changes

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/common/defaults/main.yml               |   2 +-
 roles/runtime/defaults/main.yml              |   7 +-
 roles/runtime/tasks/initialize_base.yml      |  43 ++++++++
 roles/runtime/tasks/initialize_setup_aws.yml |   7 +-
 roles/runtime/tasks/setup_aws.yml            | 109 ++++++++++++++++---
 5 files changed, 149 insertions(+), 19 deletions(-)

diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml
index 5298b5bb..9d31c7c2 100644
--- a/roles/common/defaults/main.yml
+++ b/roles/common/defaults/main.yml
@@ -121,7 +121,7 @@ common__env_name_suffix:                  "{{ env.suffix | default(common__env_s
 
 common__datalake_name:                    "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}"
 common__datalake_name_suffix:             "{{ env.datalake.suffix | default(common__datalake_suffix) }}"
-common__tunnel:                           "{{ env.tunnel | default(False) }}"
+common__tunnel:                           "{{ env.tunnel | default(True) }}"
 common__public_endpoint_access:           "{{ env.public_endpoint_access | default(not common__tunnel) }}"
 
 common__env_admin_password:               "{{ globals.admin_password | mandatory }}"
diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml
index 8cce89e2..ddbab2bf 100644
--- a/roles/runtime/defaults/main.yml
+++ b/roles/runtime/defaults/main.yml
@@ -65,7 +65,12 @@ run__ml_remove_storage:             "{{ ml.remove_storage | default (run__force_
 run__ml_public_loadbalancer:        "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}"
 
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
-run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
+run__dw_dbc_suffix:                 "{{ dw.dbc.suffix | default('dbc') }}"
+run__dw_vw_suffix:                  "{{ dw.vw.suffix | default('vw') }}"
+run__dw_tags:                       "{{ dw.tags | default(common__tags) }}"
+run__dw_overlay_network:            "{{ dw.overlay_network | bool }}"
+run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
+run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index e0bc0ade..92166c19 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -161,3 +161,46 @@
       loop_control:
         loop_var: __ml_config
         label: "{{ config.name }}"
+
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
+  block:
+    - name: Construct CDP DW Data catalog configurations
+      ansible.builtin.set_fact:
+        run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}"
+        default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}"
+        config:
+          name: "{{ __dw_config.name | default(default_dbc_name) }}"
+          load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
+          use_default_dbc: "{{ use_default_dbc }}"
+          virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
+      loop: "{{ run__dw_definitions }}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_config_index
+
+    - name: Construct CDP DW Virtual warehouse configurations
+      ansible.builtin.set_fact:
+        run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
+      vars:
+        dbc_name: "{{ item.0.name }}"
+        use_default_dbc: "{{ item.0.use_default_dbc }}"
+        config:
+          name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          dbc_name: "{{ dbc_name }}"
+          use_default_dbc: "{{ use_default_dbc }}"
+          vw_type: "{{ item.1.vw_type | default('hive') }}"
+          template: "{{ item.1.template | default('xsmall') }}"
+          autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}"
+          autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}"
+          common_configs: "{{ item.1.configs.common_configs | default({}) }}"
+          application_configs: "{{ item.1.configs.application_configs | default({}) }}"
+          ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}"
+          enable_sso: "{{ item.1.configs.enable_sso | default(None) }}"
+          tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}"
+      loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
+      loop_control:
+        index_var: __dw_dbc_index
diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml
index 69d417a2..5473311a 100644
--- a/roles/runtime/tasks/initialize_setup_aws.yml
+++ b/roles/runtime/tasks/initialize_setup_aws.yml
@@ -49,10 +49,15 @@
         label: "{{ __aws_private_subnet_item.subnet_id }}"
       loop: "{{ __aws_private_subnets_info.subnets }}"
 
+    - name: Assign AWS Private Subnet IDs in not assinged
+      when: run__datahub_private_subnet_ids is undefined
+      ansible.builtin.set_fact:
+        run__datahub_private_subnet_ids: "{{ [] }}"
+
     - name: Set fact for AWS Subnet IDs
       when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined
       ansible.builtin.set_fact:
-        run__datahub_subnet_ids: "{{ run__datahub_public_subnet_ids | default([]) | union(run__datahub_private_subnet_ids) }}"
+        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}"
 
 - name: Set fact for AWS Subnet IDs by assignment
   when: infra__aws_subnet_ids
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
index 2c2132d4..f537ded1 100644
--- a/roles/runtime/tasks/setup_aws.yml
+++ b/roles/runtime/tasks/setup_aws.yml
@@ -37,27 +37,104 @@
 - name: Setup CDP DW cluster on AWS
   when: run__include_dw
   block:
-    - name: Execute CDP DW cluster setup
-      cloudera.cloud.dw_cluster:
+#    - name: Execute CDP DW cluster setup
+#      cloudera.cloud.dw_cluster:
+#        env: "{{ run__env_name }}"
+#        overlay: "{{ run__dw_overlay_network }}"
+#        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
+#        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+#        state: present
+#        wait: yes
+#      async: 3600 # 1 hour timeout
+#      poll: 0
+#      register: __dw_builds
+#
+#    - name: Wait for CDP DW cluster setup to complete
+#      ansible.builtin.async_status:
+#        jid: "{{ __dw_builds.ansible_job_id }}"
+#      #loop_control:
+#      #  loop_var: __opdb_build
+#      #  label: "{{ __opdb_build.__opdb_config.name }}"
+#      #loop: "{{ __opdb_builds.results }}"
+#      register: __dw_builds_async
+#      until: __dw_builds_async.finished
+#      retries: 120
+#      delay: 30
+
+    - name: Retrieve CDP DW experiences
+      cloudera.cloud.dw_cluster_info:
         env: "{{ run__env_name }}"
-        overlay: no
-        # TODO - Allow direct assignment (will need to coordinate with infra role)
-        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
-        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+      register: run__dw_list
+
+    - name: Create CDP DW Database catalogs
+      when: not __dw_dbc_config.use_default_dbc
+      cloudera.cloud.dw_dbc:
+        cluster_id : "{{ run__dw_list.clusters[0].id }}"
+        name: "{{ __dw_dbc_config.name }}"
+        load_demo_data: "{{ __dw_dbc_config.load_demo_data }}"
         state: present
         wait: yes
       async: 3600 # 1 hour timeout
       poll: 0
-      register: __dw_builds
+      loop: "{{ run__dw_dbc_configs }}"
+      loop_control:
+        loop_var: __dw_dbc_config
+      register: __dw_dbc_builds
 
-    - name: Wait for CDP DW cluster setup to complete
+    - name: Wait for CDP DW Database catalogs setup to complete
+      when: __dw_dbc_build.ansible_job_id is defined
       ansible.builtin.async_status:
-        jid: "{{ __dw_builds.ansible_job_id }}"
-      #loop_control:
-      #  loop_var: __opdb_build
-      #  label: "{{ __opdb_build.__opdb_config.name }}"
-      #loop: "{{ __opdb_builds.results }}"
-      register: __dw_builds_async
-      until: __dw_builds_async.finished
+        jid: "{{ __dw_dbc_build.ansible_job_id }}"
+      register: __dw_dbc_builds_async
+      until: __dw_dbc_builds_async.finished
       retries: 120
-      delay: 30
\ No newline at end of file
+      delay: 30
+      loop: "{{ __dw_dbc_builds.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build
+
+    - name: Set CDP DW Database catalog name to id map
+      when: __dw_dbc_build_async.dbcs is defined
+      ansible.builtin.set_fact:
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}"
+      loop: "{{ __dw_dbc_builds_async.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build_async
+
+    - name: Create CDP DW Virtual warehouse
+      cloudera.cloud.dw_vw:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+        dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}"
+        vw_type: "{{ __dw_vw_config.vw_type }}"
+        name: "{{ __dw_vw_config.name }}"
+        template: "{{ __dw_vw_config.template }}"
+        autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}"
+        autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}"
+        common_configs: "{{ __dw_vw_config.common_configs }}"
+        application_configs: "{{ __dw_vw_config.application_configs }}"
+        ldap_groups: "{{ __dw_vw_config.ldap_groups }}"
+        enable_sso: "{{ __dw_vw_config.enable_sso | bool }}"
+        tags: "{{ __dw_vw_config.tags }}"
+        wait: yes
+      async: 3600 # 1 hour timeout
+      poll: 0
+      register: __dw_vw_builds
+      loop: "{{ run__dw_vw_configs }}"
+      loop_control:
+        loop_var: __dw_vw_config
+
+    - name: Wait for CDP DW Virtual warehouse setup to complete
+      when: __dw_vw_build.ansible_job_id is defined
+      ansible.builtin.async_status:
+        jid: "{{ __dw_vw_build.ansible_job_id }}"
+      register: __dw_vw_builds_async
+      until: __dw_vw_builds_async.finished
+      retries: 120
+      delay: 30
+      loop: "{{ __dw_vw_builds.results }}"
+      loop_control:
+        loop_var: __dw_vw_build
+
+    - name: Print Virtual Warehouses
+      ansible.builtin.debug:
+        msg: "Database Cluster : {{ __dw_vw_builds_async }}"
\ No newline at end of file

From a113082dc6e0b5d677e3671c3a98fb1433b85061 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Thu, 15 Jul 2021 12:26:13 +0530
Subject: [PATCH 02/10]   Revert unnecessary changes

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/common/defaults/main.yml    |  2 +-
 roles/runtime/tasks/setup_aws.yml | 46 +++++++++++++++----------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml
index 9d31c7c2..5298b5bb 100644
--- a/roles/common/defaults/main.yml
+++ b/roles/common/defaults/main.yml
@@ -121,7 +121,7 @@ common__env_name_suffix:                  "{{ env.suffix | default(common__env_s
 
 common__datalake_name:                    "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}"
 common__datalake_name_suffix:             "{{ env.datalake.suffix | default(common__datalake_suffix) }}"
-common__tunnel:                           "{{ env.tunnel | default(True) }}"
+common__tunnel:                           "{{ env.tunnel | default(False) }}"
 common__public_endpoint_access:           "{{ env.public_endpoint_access | default(not common__tunnel) }}"
 
 common__env_admin_password:               "{{ globals.admin_password | mandatory }}"
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
index f537ded1..ba40fea6 100644
--- a/roles/runtime/tasks/setup_aws.yml
+++ b/roles/runtime/tasks/setup_aws.yml
@@ -37,29 +37,29 @@
 - name: Setup CDP DW cluster on AWS
   when: run__include_dw
   block:
-#    - name: Execute CDP DW cluster setup
-#      cloudera.cloud.dw_cluster:
-#        env: "{{ run__env_name }}"
-#        overlay: "{{ run__dw_overlay_network }}"
-#        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
-#        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
-#        state: present
-#        wait: yes
-#      async: 3600 # 1 hour timeout
-#      poll: 0
-#      register: __dw_builds
-#
-#    - name: Wait for CDP DW cluster setup to complete
-#      ansible.builtin.async_status:
-#        jid: "{{ __dw_builds.ansible_job_id }}"
-#      #loop_control:
-#      #  loop_var: __opdb_build
-#      #  label: "{{ __opdb_build.__opdb_config.name }}"
-#      #loop: "{{ __opdb_builds.results }}"
-#      register: __dw_builds_async
-#      until: __dw_builds_async.finished
-#      retries: 120
-#      delay: 30
+    - name: Execute CDP DW cluster setup
+      cloudera.cloud.dw_cluster:
+        env: "{{ run__env_name }}"
+        overlay: "{{ run__dw_overlay_network }}"
+        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
+        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+        state: present
+        wait: yes
+      async: 3600 # 1 hour timeout
+      poll: 0
+      register: __dw_builds
+
+    - name: Wait for CDP DW cluster setup to complete
+      ansible.builtin.async_status:
+        jid: "{{ __dw_builds.ansible_job_id }}"
+      #loop_control:
+      #  loop_var: __opdb_build
+      #  label: "{{ __opdb_build.__opdb_config.name }}"
+      #loop: "{{ __opdb_builds.results }}"
+      register: __dw_builds_async
+      until: __dw_builds_async.finished
+      retries: 120
+      delay: 30
 
     - name: Retrieve CDP DW experiences
       cloudera.cloud.dw_cluster_info:

From d3864776bcb604e733dc7a5033a2a66f7bd94b82 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Thu, 22 Jul 2021 16:59:39 +0530
Subject: [PATCH 03/10]   Add labels for the loops

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/tasks/initialize_base.yml |  3 ++-
 roles/runtime/tasks/setup_aws.yml       | 11 ++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index 92166c19..09d76cc9 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -189,7 +189,7 @@
         dbc_name: "{{ item.0.name }}"
         use_default_dbc: "{{ item.0.use_default_dbc }}"
         config:
-          name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
           dbc_name: "{{ dbc_name }}"
           use_default_dbc: "{{ use_default_dbc }}"
           vw_type: "{{ item.1.vw_type | default('hive') }}"
@@ -204,3 +204,4 @@
       loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
       loop_control:
         index_var: __dw_dbc_index
+        label: "{{ item.0.name }}"
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
index ba40fea6..6651ffda 100644
--- a/roles/runtime/tasks/setup_aws.yml
+++ b/roles/runtime/tasks/setup_aws.yml
@@ -41,8 +41,9 @@
       cloudera.cloud.dw_cluster:
         env: "{{ run__env_name }}"
         overlay: "{{ run__dw_overlay_network }}"
+        private_load_balancer: "{{ run__dw_private_load_balancer }}"
         aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
-        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+        aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}"
         state: present
         wait: yes
       async: 3600 # 1 hour timeout
@@ -79,6 +80,7 @@
       loop: "{{ run__dw_dbc_configs }}"
       loop_control:
         loop_var: __dw_dbc_config
+        label: "{{ __dw_dbc_config.name }}"
       register: __dw_dbc_builds
 
     - name: Wait for CDP DW Database catalogs setup to complete
@@ -92,6 +94,7 @@
       loop: "{{ __dw_dbc_builds.results }}"
       loop_control:
         loop_var: __dw_dbc_build
+        label: "{{ __dw_dbc_build.__dw_dbc_config.name }}"
 
     - name: Set CDP DW Database catalog name to id map
       when: __dw_dbc_build_async.dbcs is defined
@@ -122,6 +125,7 @@
       loop: "{{ run__dw_vw_configs }}"
       loop_control:
         loop_var: __dw_vw_config
+        label: "{{ __dw_vw_config.name }}"
 
     - name: Wait for CDP DW Virtual warehouse setup to complete
       when: __dw_vw_build.ansible_job_id is defined
@@ -134,7 +138,4 @@
       loop: "{{ __dw_vw_builds.results }}"
       loop_control:
         loop_var: __dw_vw_build
-
-    - name: Print Virtual Warehouses
-      ansible.builtin.debug:
-        msg: "Database Cluster : {{ __dw_vw_builds_async }}"
\ No newline at end of file
+        label: "{{ __dw_vw_build.__dw_vw_config.name }}"
\ No newline at end of file

From 8d08f521c3926ba7b79300f23cb510de93a9d0b3 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Thu, 22 Jul 2021 18:42:17 +0530
Subject: [PATCH 04/10]   Remove use_default_dbc flag

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/tasks/initialize_base.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index 09d76cc9..defdfbad 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -170,10 +170,9 @@
         run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
       vars:
         include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
-        use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}"
-        default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}"
+        use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}"
         config:
-          name: "{{ __dw_config.name | default(default_dbc_name) }}"
+          name: "{{ __dw_config.name | default('') }}"
           load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
           use_default_dbc: "{{ use_default_dbc }}"
           virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"

From b74ef0a78f1fc38e3c853427b99dd06792600bd9 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Fri, 23 Jul 2021 15:27:16 +0530
Subject: [PATCH 05/10]   Change the vw and dbc module names and other minor
 refactor

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/tasks/initialize_base.yml |  6 +++---
 roles/runtime/tasks/setup_aws.yml       | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index defdfbad..351ff73d 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -191,10 +191,10 @@
           name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
           dbc_name: "{{ dbc_name }}"
           use_default_dbc: "{{ use_default_dbc }}"
-          vw_type: "{{ item.1.vw_type | default('hive') }}"
+          type: "{{ item.1.type | default('hive') }}"
           template: "{{ item.1.template | default('xsmall') }}"
-          autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}"
-          autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}"
+          autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}"
+          autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}"
           common_configs: "{{ item.1.configs.common_configs | default({}) }}"
           application_configs: "{{ item.1.configs.application_configs | default({}) }}"
           ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}"
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
index 6651ffda..8ed1edd3 100644
--- a/roles/runtime/tasks/setup_aws.yml
+++ b/roles/runtime/tasks/setup_aws.yml
@@ -69,7 +69,7 @@
 
     - name: Create CDP DW Database catalogs
       when: not __dw_dbc_config.use_default_dbc
-      cloudera.cloud.dw_dbc:
+      cloudera.cloud.dw_database_catalog:
         cluster_id : "{{ run__dw_list.clusters[0].id }}"
         name: "{{ __dw_dbc_config.name }}"
         load_demo_data: "{{ __dw_dbc_config.load_demo_data }}"
@@ -97,22 +97,22 @@
         label: "{{ __dw_dbc_build.__dw_dbc_config.name }}"
 
     - name: Set CDP DW Database catalog name to id map
-      when: __dw_dbc_build_async.dbcs is defined
+      when: __dw_dbc_build_async.database_catalogs is defined
       ansible.builtin.set_fact:
-        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}"
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}"
       loop: "{{ __dw_dbc_builds_async.results }}"
       loop_control:
         loop_var: __dw_dbc_build_async
 
     - name: Create CDP DW Virtual warehouse
-      cloudera.cloud.dw_vw:
+      cloudera.cloud.dw_virtual_warehouse:
         cluster_id: "{{ run__dw_list.clusters[0].id }}"
         dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}"
-        vw_type: "{{ __dw_vw_config.vw_type }}"
+        type: "{{ __dw_vw_config.type }}"
         name: "{{ __dw_vw_config.name }}"
         template: "{{ __dw_vw_config.template }}"
-        autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}"
-        autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}"
+        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}"
+        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}"
         common_configs: "{{ __dw_vw_config.common_configs }}"
         application_configs: "{{ __dw_vw_config.application_configs }}"
         ldap_groups: "{{ __dw_vw_config.ldap_groups }}"

From 41c6383c32cb5dd68a3acf269c64c554c70c2304 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Tue, 27 Jul 2021 22:32:37 +0530
Subject: [PATCH 06/10]   Minor refactor

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/tasks/initialize_base.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index 351ff73d..41c42a87 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -185,12 +185,10 @@
       ansible.builtin.set_fact:
         run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
       vars:
-        dbc_name: "{{ item.0.name }}"
-        use_default_dbc: "{{ item.0.use_default_dbc }}"
         config:
           name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
-          dbc_name: "{{ dbc_name }}"
-          use_default_dbc: "{{ use_default_dbc }}"
+          dbc_name: "{{ item.0.name }}"
+          use_default_dbc: "{{ item.0.use_default_dbc }}"
           type: "{{ item.1.type | default('hive') }}"
           template: "{{ item.1.template | default('xsmall') }}"
           autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}"
@@ -203,4 +201,4 @@
       loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
       loop_control:
         index_var: __dw_dbc_index
-        label: "{{ item.0.name }}"
+        label: "{{ item.0.name }}"
\ No newline at end of file

From 84576b935b5773c12c68cb55b8a8ec3bd2dff2d3 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Mon, 30 Aug 2021 17:36:45 +0530
Subject: [PATCH 07/10]   Address review comments

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/defaults/main.yml              |  3 ++
 roles/runtime/tasks/initialize_base.yml      | 30 ++++++++++----------
 roles/runtime/tasks/initialize_setup_aws.yml | 11 ++++---
 roles/runtime/tasks/setup_aws.yml            | 23 +++++++++++----
 4 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml
index ddbab2bf..90bc67ba 100644
--- a/roles/runtime/defaults/main.yml
+++ b/roles/runtime/defaults/main.yml
@@ -72,6 +72,9 @@ run__dw_overlay_network:            "{{ dw.overlay_network | bool }}"
 run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
 run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
+run__dw_default_vw_type:            "{{ dw.default_vw_type | default('hive') }}"
+run__dw_default_template_type:      "{{ dw.default_template_type | default('xsmall') }}"
+run__dw_default_dbc_suffix:         "{{ dw.default_dbc_suffix | default('-dl-default') }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"
diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
index 41c42a87..6dca5c64 100644
--- a/roles/runtime/tasks/initialize_base.yml
+++ b/roles/runtime/tasks/initialize_base.yml
@@ -170,11 +170,10 @@
         run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
       vars:
         include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
-        use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}"
         config:
           name: "{{ __dw_config.name | default('') }}"
           load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
-          use_default_dbc: "{{ use_default_dbc }}"
+          use_default_dbc: "{{ __dw_config.name is undefined | bool }}"
           virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
       loop: "{{ run__dw_definitions }}"
       loop_control:
@@ -186,19 +185,20 @@
         run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
       vars:
         config:
-          name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
-          dbc_name: "{{ item.0.name }}"
-          use_default_dbc: "{{ item.0.use_default_dbc }}"
-          type: "{{ item.1.type | default('hive') }}"
-          template: "{{ item.1.template | default('xsmall') }}"
-          autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}"
-          autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}"
-          common_configs: "{{ item.1.configs.common_configs | default({}) }}"
-          application_configs: "{{ item.1.configs.application_configs | default({}) }}"
-          ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}"
-          enable_sso: "{{ item.1.configs.enable_sso | default(None) }}"
-          tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}"
+          name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          dbc_name: "{{ __dw_config.0.name }}"
+          use_default_dbc: "{{ __dw_config.0.use_default_dbc }}"
+          type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}"
+          template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}"
+          autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}"
+          autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}"
+          common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}"
+          application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}"
+          ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}"
+          enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}"
+          tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}"
       loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
       loop_control:
+        loop_var: __dw_config
         index_var: __dw_dbc_index
-        label: "{{ item.0.name }}"
\ No newline at end of file
+        label: "{{ __dw_config.0.name }}"
\ No newline at end of file
diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml
index 5473311a..d5519c8a 100644
--- a/roles/runtime/tasks/initialize_setup_aws.yml
+++ b/roles/runtime/tasks/initialize_setup_aws.yml
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 - name: Discover AWS Public and Private VPC Subnets
-  when: not infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is undefined
   block:
     - name: Query AWS Public Subnets
       amazon.aws.ec2_vpc_subnet_info:
@@ -49,18 +49,17 @@
         label: "{{ __aws_private_subnet_item.subnet_id }}"
       loop: "{{ __aws_private_subnets_info.subnets }}"
 
-    - name: Assign AWS Private Subnet IDs in not assinged
-      when: run__datahub_private_subnet_ids is undefined
+    - name: Set default AWS Private Subnet if there are no Private Subnets
       ansible.builtin.set_fact:
-        run__datahub_private_subnet_ids: "{{ [] }}"
+        run__datahub_private_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) }}"
 
     - name: Set fact for AWS Subnet IDs
       when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined
       ansible.builtin.set_fact:
-        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}"
+        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | union(run__datahub_public_subnet_ids) }}"
 
 - name: Set fact for AWS Subnet IDs by assignment
-  when: infra__aws_subnet_ids
+  when: infra__aws_subnet_ids is defined
   block:
     - name: Set fact for All AWS Subnet IDs by assignment
       ansible.builtin.set_fact:
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
index 8ed1edd3..ccd17f11 100644
--- a/roles/runtime/tasks/setup_aws.yml
+++ b/roles/runtime/tasks/setup_aws.yml
@@ -67,6 +67,19 @@
         env: "{{ run__env_name }}"
       register: run__dw_list
 
+    - name: Fetch all the Database Catalogs under the cluster
+      cloudera.cloud.dw_database_catalog_info:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+      register: __dbc_list
+
+    - name: Set default Database catalog id for the cluster
+      when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$')
+      ansible.builtin.set_fact:
+        __default_dbc_id: "{{ __dw_dbc.id }}"
+      loop: "{{ __dbc_list.database_catalogs }}"
+      loop_control:
+        loop_var: __dw_dbc
+
     - name: Create CDP DW Database catalogs
       when: not __dw_dbc_config.use_default_dbc
       cloudera.cloud.dw_database_catalog:
@@ -97,9 +110,9 @@
         label: "{{ __dw_dbc_build.__dw_dbc_config.name }}"
 
     - name: Set CDP DW Database catalog name to id map
-      when: __dw_dbc_build_async.database_catalogs is defined
+      when: __dw_dbc_build_async.database_catalog is defined
       ansible.builtin.set_fact:
-        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}"
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}"
       loop: "{{ __dw_dbc_builds_async.results }}"
       loop_control:
         loop_var: __dw_dbc_build_async
@@ -107,12 +120,12 @@
     - name: Create CDP DW Virtual warehouse
       cloudera.cloud.dw_virtual_warehouse:
         cluster_id: "{{ run__dw_list.clusters[0].id }}"
-        dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}"
+        dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}"
         type: "{{ __dw_vw_config.type }}"
         name: "{{ __dw_vw_config.name }}"
         template: "{{ __dw_vw_config.template }}"
-        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}"
-        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}"
+        autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}"
+        autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}"
         common_configs: "{{ __dw_vw_config.common_configs }}"
         application_configs: "{{ __dw_vw_config.application_configs }}"
         ldap_groups: "{{ __dw_vw_config.ldap_groups }}"

From bf69eeda38dc0a219956e775916b6c7827cc56d3 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Tue, 21 Sep 2021 02:19:15 +0530
Subject: [PATCH 08/10]   Refactor cdw teardown logic

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/runtime/tasks/teardown_base.yml | 151 +++++++++++++++++++++++---
 1 file changed, 135 insertions(+), 16 deletions(-)

diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml
index a5707698..b5776ef4 100644
--- a/roles/runtime/tasks/teardown_base.yml
+++ b/roles/runtime/tasks/teardown_base.yml
@@ -32,17 +32,90 @@
   poll: 0
   register: __opdb_teardowns_info
 
-- name: Execute CDP DW cluster teardown
-  register: __dw_teardown_info
+- name: Execute CDP DW teardown
   when:
-    - run__include_dw or run__force_teardown | bool
+    - run__include_dw | bool
     - run__env_info.environments | length > 0
     - run__env_info.environments[0].descendants.dw | length > 0
-  cloudera.cloud.dw_cluster:
-    env: "{{ run__env_name }}"
-    state: absent
-    wait: no
-    force: "{{ run__dw_force_delete }}"
+  block:
+    - name: Execute CDP DW virtual warehouse teardown
+      when: run__force_teardown | bool
+      cloudera.cloud.dw_cluster:
+        env: "{{ run__env_name }}"
+        state: absent
+        wait: no
+        force: "{{ run__dw_force_delete }}"
+      register: __dw_cluster_teardown_info
+
+    - name: Execute CDP DW virtual warehouse teardown
+      when: not run__force_teardown | bool
+      block:
+        - name: Retrieve CDP DW experiences
+          cloudera.cloud.dw_cluster_info:
+            env: "{{ run__env_name }}"
+          register: __dw_list
+
+        - name: Debug output CDP DW
+          debug:
+            msg: "CDP DW : {{ __dw_list }}"
+
+        - name: Fetch all the CDP DW Database Catalogs under the cluster
+          cloudera.cloud.dw_database_catalog_info:
+            cluster_id: "{{ __dw_list.clusters[0].id }}"
+          register: __dbc_list
+
+        - name: Debug output CDP DW DBCs
+          debug:
+            msg: "CDP DW DBC : {{ __dbc_list }}"
+
+        - name: Fetch all the CDP DW Virtual Warehouse
+          cloudera.cloud.dw_virtual_warehouse_info:
+            cluster_id: "{{ __dw_list.clusters[0].id }}"
+            dbc_id: "{{ __dbc.id }}"
+          loop: "{{ __dbc_list.database_catalogs }}"
+          loop_control:
+            loop_var: __dbc
+          register: __vw_list
+
+        - name: Debug output CDP DW VWs
+          debug:
+            msg: "CDP DW VWs : {{ __vw_list }}"
+#
+#        - name: Delete all CDP DW Virtual Warehouses
+#          cloudera.cloud.dw_virtual_warehouse:
+#            cluster_id: "{{ __dw_list.clusters[0].id }}"
+#            id: "{{ __vw.id }}"
+#            state: absent
+#            wait: no
+#          loop: "{{ __vw_list.results[0].virtual_warehouses }}"
+#          loop_control:
+#            loop_var: __vw
+#          async: 3600 # 1 hour timeout
+#          poll: 0
+#          register: __dw_vw_teardown_info
+
+#        - name: Delete all CDP DW Database Catalogs
+#          cloudera.cloud.dw_database_catalog:
+#            cluster_id: "{{ __dw_list.clusters[0].id }}"
+#            name: "{{ __dbc.name }}"
+#            state: absent
+#            wait: no
+#          loop: "{{ __dbc_list.database_catalogs }}"
+#          loop_control:
+#            loop_var: __dbc
+#          async: 3600 # 1 hour timeout
+#          poll: 0
+#          register: __dw_dbc_teardown_info
+
+        - name: Execute CDP DW virtual warehouse teardown
+          cloudera.cloud.dw_cluster:
+            env: "{{ run__env_name }}"
+            state: absent
+
+            wait: no
+          async: 3600 # 1 hour timeout
+          poll: 0
+          register: __dw_cluster_teardown_info
 
 - name: Execute CDP Dataflow teardown
   register: __df_teardown_info
@@ -130,14 +203,60 @@
   delay: 30
 
 - name: Wait for CDP DW deployments to decommission
-  when:
-    - __dw_teardown_info is defined
-    - __dw_teardown_info.started | default(False)
-  cloudera.cloud.dw_cluster:
-    env: "{{ run__env_name }}"
-    state: absent
-    wait: yes
-    force: "{{ run__dw_force_delete }}"
+  when: run__include_dw | bool
+  block:
+#    - name: CDW cluster teardown info
+#      debug:
+#        msg: "CDW : {{ __dw_vw_teardown_info }}"
+#
+#    - name: Wait for CDW Virtual Warehouse to decommision
+#      when:
+#      - __dw_vw_teardown_info is defined
+#      - __dw_vw_teardown_info.results is defined
+#      - __dw_vw_teardown_info.results | length > 0
+#      ansible.builtin.async_status:
+#        jid: "{{ __dw_vw_teardown.ansible_job_id }}"
+#      loop_control:
+#        loop_var: __dw_vw_teardown
+#      loop: "{{ __dw_vw_teardown_info.results }}"
+#      register: __dw_vw_teardown_async
+#      until: __dw_vw_teardown_async.finished
+#      retries: 120
+#      delay: 30
+#
+#    - name: Wait for CDW Database Catalog to decommision
+#      when:
+#        - __dw_dbc_teardown_info is defined
+#        - __dw_dbc_teardown_info.results is defined
+#        - __dw_dbc_teardown_info.results | length > 0
+#      ansible.builtin.async_status:
+#        jid: "{{ __dw_dbc_teardown.ansible_job_id }}"
+#      loop_control:
+#        loop_var: __dw_dbc_teardown
+#      loop: "{{ __dw_dbc_teardown_info.results }}"
+#      register: __dw_dbc_teardown_async
+#      until: __dw_dbc_teardown_async.finished
+#      retries: 120
+#      delay: 30
+
+    - name: CDW cluster teardown info
+      debug:
+        msg: "CDW : {{ __dw_cluster_teardown_info }}"
+
+    - name: Wait for CDW cluster to decommision
+      when:
+        - __dw_cluster_teardown_info is defined
+        - __dw_cluster_teardown_info.results is defined
+        - __dw_cluster_teardown_info.results | length > 0
+      ansible.builtin.async_status:
+        jid: "{{ __dw_cluster_teardown.ansible_job_id }}"
+      loop_control:
+        loop_var: __dw_cluster_teardown
+      loop: "{{ __dw_cluster_teardown_info }}"
+      register: __dw_cluster_teardown_async
+      until: __dw_cluster_teardown_async.finished
+      retries: 120
+      delay: 30
 
 - name: Wait for CDP OpDB deployments to decommission
   when:

From ababc1e552be812fd3f2d3ef14cd9af0a7ed479b Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Wed, 22 Sep 2021 19:11:58 +0530
Subject: [PATCH 09/10]   Add support for CDW partial teardown

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/common/defaults/main.yml              |   3 +
 roles/runtime/defaults/main.yml             |   1 +
 roles/runtime/tasks/initialize_teardown.yml | 109 +++++++++-
 roles/runtime/tasks/teardown_base.yml       | 210 +++++++-------------
 roles/sequence/defaults/main.yml            |  14 +-
 roles/sequence/tasks/main.yml               |  10 +
 6 files changed, 207 insertions(+), 140 deletions(-)

diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml
index 5298b5bb..e9620485 100644
--- a/roles/common/defaults/main.yml
+++ b/roles/common/defaults/main.yml
@@ -132,6 +132,9 @@ common__setup_runtime:                    "{{ ml is defined or de is defined or
 common__setup_plat:                       "{{ env is defined or sequence__setup_runtime  | default(False) | bool }}"
 common__setup_infra:                      "{{ infra is defined or sequence__setup_plat | default(False) | bool }}"
 common__setup_base:                       "{{ mgmt is defined or clusters is defined | default(False) | bool }}"
+common__teardown_runtime:                 "{{ common__setup_runtime }}"
+common__teardown_plat:                    "{{ common__setup_plat and dw.teardown is undefined }}"
+common__teardown_infra:                   "{{ common__setup_infra and dw.teardown is undefined }}"
 
 common__include_ml:                        "{{ ml is defined | bool }}"
 common__include_dw:                        "{{ dw is defined | bool }}"
diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml
index 90bc67ba..55c2a1d8 100644
--- a/roles/runtime/defaults/main.yml
+++ b/roles/runtime/defaults/main.yml
@@ -75,6 +75,7 @@ run__dw_force_delete:               "{{ dw.force_delete | default (run__force_te
 run__dw_default_vw_type:            "{{ dw.default_vw_type | default('hive') }}"
 run__dw_default_template_type:      "{{ dw.default_template_type | default('xsmall') }}"
 run__dw_default_dbc_suffix:         "{{ dw.default_dbc_suffix | default('-dl-default') }}"
+run__dw_teardown:                   "{{ dw.teardown | default(omit) }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"
diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml
index b798bfa8..ec6b7cdf 100644
--- a/roles/runtime/tasks/initialize_teardown.yml
+++ b/roles/runtime/tasks/initialize_teardown.yml
@@ -68,4 +68,111 @@
       loop: "{{ run__env_info.environments[0].descendants.ml }}"
       loop_control:
         loop_var: __ml_config
-        label: "{{ __ml_config.instanceName }}"
\ No newline at end of file
+        label: "{{ __ml_config.instanceName }}"
+
+- name: Discover CDW deployments
+  when: run__include_dw
+  block:
+    - name: Retrieve CDP DW experiences
+      cloudera.cloud.dw_cluster_info:
+        env: "{{ run__env_name }}"
+      register: __dw_list
+
+    - name: Initialize CDP DW cluster id
+      ansible.builtin.set_fact:
+        run__dw_cluster_id: "{{ __dw_list.clusters[0].id }}"
+
+    - name: Prepare partial teardown for CDW resources
+      when: run__dw_teardown is defined
+      block:
+        - name: Initialize list of CDW database catalogs to be decommissioned
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ [] }}"
+
+        - name: Initialize list of CDW virtual warehouses to be decommissioned
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ [] }}"
+
+        - name: Populate CDW database catalogs defined with its id to the teardown list
+          when: __dbc.id is defined
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc.id ] }}"
+          loop: "{{ run__dw_teardown.database_catalogs }}"
+          loop_control:
+            loop_var: __dbc
+            label: "{{ __dbc.id }}"
+
+        - name: Fetch the ids of all the CDW database catalogs defined with its name
+          when: __dbc.name is defined
+          cloudera.cloud.dw_database_catalog_info:
+             name: "{{ __dbc.name }}"
+             cluster_id: "{{ run__dw_cluster_id }}"
+          loop: "{{ run__dw_teardown.database_catalogs }}"
+          loop_control:
+            loop_var: __dbc
+            label: "{{ __dbc.name }}"
+          register: __dbc_name_list
+
+        - name: Populate CDW database catalogs defined with its name to the teardown list
+          when:
+              - __dbc_result.database_catalogs is defined
+              - __dbc_result.database_catalogs | length > 0
+              - __dbc_result.database_catalogs[0].id not in run__dw_dbc_ids
+          ansible.builtin.set_fact:
+            run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc_result.database_catalogs[0].id ] }}"
+          loop: "{{ __dbc_name_list.results }}"
+          loop_control:
+            loop_var: __dbc_result
+            label: "{{ __dbc_result.database_catalogs[0].id }}"
+
+        - name: Fetch all the CDW virtual warehouses under a CDW cluster
+          cloudera.cloud.dw_virtual_warehouse_info:
+            cluster_id: "{{ run__dw_cluster_id }}"
+          register: __all_vw_list
+
+        - name: Populate all the CDW virtual warehouses of database catalogs scheduled for teardown to the teardown list
+          when:
+              - run__dw_dbc_ids is defined
+              - __vw.dbcId in run__dw_dbc_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ] }}"
+          loop: "{{ __all_vw_list.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.id }}"
+
+        - name: Fetch the ids of all the CDW virtual warehouses defined with its name
+          when:
+               - run__dw_teardown.virtual_warehouses is defined
+               - __vw.name is defined
+          cloudera.cloud.dw_virtual_warehouse_info:
+            name: "{{ __vw.name }}"
+            cluster_id: "{{ run__dw_cluster_id }}"
+          loop: "{{ run__dw_teardown.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.name }}"
+          register: __vw_name_list
+
+        - name: Populate CDW virtual warehouses defined with its id to the teardown list
+          when:
+               - __vw.id is defined
+               - __vw.id not in run__dw_vw_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ]}}"
+          loop: "{{ run__dw_teardown.virtual_warehouses }}"
+          loop_control:
+            loop_var: __vw
+            label: "{{ __vw.id }}"
+
+        - name: Populate CDW virtual warehouses defined with its name to the teardown list
+          when:
+               - __vw_result.virtual_warehouses is defined
+               - __vw_result.virtual_warehouses | length > 0
+               - __vw_result.virtual_warehouses[0].id not in run__dw_vw_ids
+          ansible.builtin.set_fact:
+            run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw_result.virtual_warehouses[0].id ]}}"
+          loop: "{{ __vw_name_list.results }}"
+          loop_control:
+            loop_var: __vw_result
+            label: "{{ __vw_result.virtual_warehouses[0].id }}"
diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml
index b5776ef4..8beeb2ec 100644
--- a/roles/runtime/tasks/teardown_base.yml
+++ b/roles/runtime/tasks/teardown_base.yml
@@ -33,89 +33,32 @@
   register: __opdb_teardowns_info
 
 - name: Execute CDP DW teardown
-  when:
-    - run__include_dw | bool
-    - run__env_info.environments | length > 0
-    - run__env_info.environments[0].descendants.dw | length > 0
-  block:
-    - name: Execute CDP DW virtual warehouse teardown
-      when: run__force_teardown | bool
-      cloudera.cloud.dw_cluster:
-        env: "{{ run__env_name }}"
-        state: absent
-        wait: no
-        force: "{{ run__dw_force_delete }}"
-      register: __dw_cluster_teardown_info
-
-    - name: Execute CDP DW virtual warehouse teardown
-      when: not run__force_teardown | bool
-      block:
-        - name: Retrieve CDP DW experiences
-          cloudera.cloud.dw_cluster_info:
-            env: "{{ run__env_name }}"
-          register: __dw_list
-
-        - name: Debug output CDP DW
-          debug:
-            msg: "CDP DW : {{ __dw_list }}"
-
-        - name: Fetch all the CDP DW Database Catalogs under the cluster
-          cloudera.cloud.dw_database_catalog_info:
-            cluster_id: "{{ __dw_list.clusters[0].id }}"
-          register: __dbc_list
-
-        - name: Debug output CDP DW DBCs
-          debug:
-            msg: "CDP DW DBC : {{ __dbc_list }}"
-
-        - name: Fetch all the CDP DW Virtual Warehouse
-          cloudera.cloud.dw_virtual_warehouse_info:
-            cluster_id: "{{ __dw_list.clusters[0].id }}"
-            dbc_id: "{{ __dbc.id }}"
-          loop: "{{ __dbc_list.database_catalogs }}"
-          loop_control:
-            loop_var: __dbc
-          register: __vw_list
-
-        - name: Debug output CDP DW VWs
-          debug:
-            msg: "CDP DW VWs : {{ __vw_list }}"
-#
-#        - name: Delete all CDP DW Virtual Warehouses
-#          cloudera.cloud.dw_virtual_warehouse:
-#            cluster_id: "{{ __dw_list.clusters[0].id }}"
-#            id: "{{ __vw.id }}"
-#            state: absent
-#            wait: no
-#          loop: "{{ __vw_list.results[0].virtual_warehouses }}"
-#          loop_control:
-#            loop_var: __vw
-#          async: 3600 # 1 hour timeout
-#          poll: 0
-#          register: __dw_vw_teardown_info
-
-#        - name: Delete all CDP DW Database Catalogs
-#          cloudera.cloud.dw_database_catalog:
-#            cluster_id: "{{ __dw_list.clusters[0].id }}"
-#            name: "{{ __dbc.name }}"
-#            state: absent
-#            wait: no
-#          loop: "{{ __dbc_list.database_catalogs }}"
-#          loop_control:
-#            loop_var: __dbc
-#          async: 3600 # 1 hour timeout
-#          poll: 0
-#          register: __dw_dbc_teardown_info
-
-        - name: Execute CDP DW virtual warehouse teardown
-          cloudera.cloud.dw_cluster:
-            env: "{{ run__env_name }}"
-            state: absent
+  when: run__dw_teardown is not defined
+  cloudera.cloud.dw_cluster:
+    env: "{{ run__env_name }}"
+    state: absent
+    wait: yes
+    force: "{{ run__dw_force_delete }}"
+  async: 3600 # 1 hour timeout
+  poll: 0
+  register: __dw_cluster_teardown_info
 
-            wait: no
-          async: 3600 # 1 hour timeout
-          poll: 0
-          register: __dw_cluster_teardown_info
+- name: Execute CDP virtual warehouse partial teardown
+  when:
+       - run__dw_teardown is defined
+       - run__dw_vw_ids is defined
+       - run__dw_vw_ids | length > 0
+  cloudera.cloud.dw_virtual_warehouse:
+    cluster_id: "{{ run__dw_cluster_id }}"
+    id: "{{ __vw_id }}"
+    state: absent
+    wait: yes
+  loop: "{{ run__dw_vw_ids }}"
+  loop_control:
+    loop_var: __vw_id
+  async: 3600 # 1 hour timeout
+  poll: 0
+  register: __dw_vw_teardown_info
 
 - name: Execute CDP Dataflow teardown
   register: __df_teardown_info
@@ -202,61 +145,58 @@
   retries: 120
   delay: 30
 
-- name: Wait for CDP DW deployments to decommission
-  when: run__include_dw | bool
-  block:
-#    - name: CDW cluster teardown info
-#      debug:
-#        msg: "CDW : {{ __dw_vw_teardown_info }}"
-#
-#    - name: Wait for CDW Virtual Warehouse to decommision
-#      when:
-#      - __dw_vw_teardown_info is defined
-#      - __dw_vw_teardown_info.results is defined
-#      - __dw_vw_teardown_info.results | length > 0
-#      ansible.builtin.async_status:
-#        jid: "{{ __dw_vw_teardown.ansible_job_id }}"
-#      loop_control:
-#        loop_var: __dw_vw_teardown
-#      loop: "{{ __dw_vw_teardown_info.results }}"
-#      register: __dw_vw_teardown_async
-#      until: __dw_vw_teardown_async.finished
-#      retries: 120
-#      delay: 30
-#
-#    - name: Wait for CDW Database Catalog to decommision
-#      when:
-#        - __dw_dbc_teardown_info is defined
-#        - __dw_dbc_teardown_info.results is defined
-#        - __dw_dbc_teardown_info.results | length > 0
-#      ansible.builtin.async_status:
-#        jid: "{{ __dw_dbc_teardown.ansible_job_id }}"
-#      loop_control:
-#        loop_var: __dw_dbc_teardown
-#      loop: "{{ __dw_dbc_teardown_info.results }}"
-#      register: __dw_dbc_teardown_async
-#      until: __dw_dbc_teardown_async.finished
-#      retries: 120
-#      delay: 30
+- name: Wait for CDP DW virtual warehouse to decommission
+  when:
+       - __dw_vw_teardown_info is defined
+       - __dw_vw_teardown_info.results is defined
+       - __dw_vw_teardown_info.results | length > 0
+  ansible.builtin.async_status:
+    jid: "{{ __dw_vw_teardown.ansible_job_id }}"
+  loop_control:
+    loop_var: __dw_vw_teardown
+  loop: "{{ __dw_vw_teardown_info.results }}"
+  register: __dw_vw_teardown_async
+  until: __dw_vw_teardown_async.finished
+  retries: 120
+  delay: 30
+
+- name: Execute CDP DW Database Catalogs partial teardown
+  when: run__dw_teardown.database_catalogs is defined
+  cloudera.cloud.dw_database_catalog:
+     cluster_id: "{{ run__dw_cluster_id }}"
+     id: "{{ __dbc_id }}"
+     state: absent
+     wait: yes
+  loop: "{{ run__dw_dbc_ids }}"
+  loop_control:
+     loop_var: __dbc_id
+  async: 3600 # 1 hour timeout
+  poll: 0
+  register: __dw_dbc_teardown_info
 
-    - name: CDW cluster teardown info
-      debug:
-        msg: "CDW : {{ __dw_cluster_teardown_info }}"
+- name: Wait for CDW Database Catalog to decommision
+  when:
+       - __dw_dbc_teardown_info is defined
+       - __dw_dbc_teardown_info.results is defined
+       - __dw_dbc_teardown_info.results | length > 0
+  ansible.builtin.async_status:
+    jid: "{{ __dw_dbc_teardown.ansible_job_id }}"
+  loop_control:
+    loop_var: __dw_dbc_teardown
+  loop: "{{ __dw_dbc_teardown_info.results }}"
+  register: __dw_dbc_teardown_async
+  until: __dw_dbc_teardown_async.finished
+  retries: 120
+  delay: 30
 
-    - name: Wait for CDW cluster to decommision
-      when:
-        - __dw_cluster_teardown_info is defined
-        - __dw_cluster_teardown_info.results is defined
-        - __dw_cluster_teardown_info.results | length > 0
-      ansible.builtin.async_status:
-        jid: "{{ __dw_cluster_teardown.ansible_job_id }}"
-      loop_control:
-        loop_var: __dw_cluster_teardown
-      loop: "{{ __dw_cluster_teardown_info }}"
-      register: __dw_cluster_teardown_async
-      until: __dw_cluster_teardown_async.finished
-      retries: 120
-      delay: 30
+- name: Wait for CDW cluster to decommision
+  when: __dw_cluster_teardown_info.ansible_job_id is defined
+  ansible.builtin.async_status:
+    jid: "{{ __dw_cluster_teardown_info.ansible_job_id }}"
+  register: __dw_cluster_teardown_async
+  until: __dw_cluster_teardown_async.finished
+  retries: 120
+  delay: 30
 
 - name: Wait for CDP OpDB deployments to decommission
   when:
diff --git a/roles/sequence/defaults/main.yml b/roles/sequence/defaults/main.yml
index 598301db..7fc8f3b2 100644
--- a/roles/sequence/defaults/main.yml
+++ b/roles/sequence/defaults/main.yml
@@ -12,10 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
-sequence__setup_runtime:  "{{ common__setup_runtime }}"
+sequence__setup_runtime:    "{{ common__setup_runtime }}"
 
-sequence__setup_plat:     "{{ common__setup_plat }}"
+sequence__setup_plat:       "{{ common__setup_plat }}"
 
-sequence__setup_infra:    "{{ common__setup_infra }}"
+sequence__setup_infra:      "{{ common__setup_infra }}"
 
-sequence_init:            "{{ sequence__setup_infra }}"
\ No newline at end of file
+sequence_init:              "{{ sequence__setup_infra }}"
+
+sequence__teardown_runtime: "{{ common__teardown_runtime }}"
+
+sequence__teardown_plat:    "{{ common__teardown_plat }}"
+
+sequence__teardown_infra:   "{{ common__teardown_infra }}"
\ No newline at end of file
diff --git a/roles/sequence/tasks/main.yml b/roles/sequence/tasks/main.yml
index b77214fe..eca1e6c1 100644
--- a/roles/sequence/tasks/main.yml
+++ b/roles/sequence/tasks/main.yml
@@ -59,6 +59,7 @@
     - run
 
 - name: Initialize Runtime Teardown
+  when: sequence__teardown_runtime | bool
   ansible.builtin.include_role:
     name: cloudera.exe.runtime
     tasks_from: initialize_teardown
@@ -74,6 +75,7 @@
     - never
 
 - name: Teardown Runtime
+  when: sequence__teardown_runtime | bool
   ansible.builtin.include_role:
     name: cloudera.exe.runtime
     tasks_from: teardown
@@ -88,7 +90,12 @@
     - plat
     - never
 
+- name: CDW cluster teardown info
+  debug:
+     msg: "Sequence teardown platform : {{ sequence__teardown_plat }}"
+
 - name: Initialize Platform Teardown
+  when: sequence__teardown_plat | bool
   ansible.builtin.include_role:
     name: cloudera.exe.platform
     tasks_from: initialize_teardown
@@ -102,6 +109,7 @@
     - never
 
 - name: Teardown Platform
+  when: sequence__teardown_plat | bool
   ansible.builtin.include_role:
     name: cloudera.exe.platform
     tasks_from: teardown
@@ -115,6 +123,7 @@
     - never
 
 - name: Initialize Infrastructure Teardown
+  when: sequence__teardown_infra | bool
   ansible.builtin.include_role:
     name: cloudera.exe.infrastructure
     tasks_from: initialize_teardown
@@ -126,6 +135,7 @@
     - never
 
 - name: Teardown Infrastructure
+  when: sequence__teardown_infra | bool
   ansible.builtin.include_role:
     name: cloudera.exe.infrastructure
     tasks_from: teardown

From a85ea21d3d6b9f30d1a23f448d5a19bf66f37b83 Mon Sep 17 00:00:00 2001
From: Saravanan Raju <saravanan.footloose@gmail.com>
Date: Wed, 22 Sep 2021 19:14:57 +0530
Subject: [PATCH 10/10]   Remove logs from code

Signed-off-by: Saravanan Raju <saravanan.footloose@gmail.com>
---
 roles/sequence/tasks/main.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/roles/sequence/tasks/main.yml b/roles/sequence/tasks/main.yml
index eca1e6c1..45b7ecff 100644
--- a/roles/sequence/tasks/main.yml
+++ b/roles/sequence/tasks/main.yml
@@ -90,10 +90,6 @@
     - plat
     - never
 
-- name: CDW cluster teardown info
-  debug:
-     msg: "Sequence teardown platform : {{ sequence__teardown_plat }}"
-
 - name: Initialize Platform Teardown
   when: sequence__teardown_plat | bool
   ansible.builtin.include_role: