diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5298b5bb..e9620485 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -132,6 +132,9 @@ common__setup_runtime: "{{ ml is defined or de is defined or common__setup_plat: "{{ env is defined or sequence__setup_runtime | default(False) | bool }}" common__setup_infra: "{{ infra is defined or sequence__setup_plat | default(False) | bool }}" common__setup_base: "{{ mgmt is defined or clusters is defined | default(False) | bool }}" +common__teardown_runtime: "{{ common__setup_runtime }}" +common__teardown_plat: "{{ common__setup_plat and dw.teardown is undefined }}" +common__teardown_infra: "{{ common__setup_infra and dw.teardown is undefined }}" common__include_ml: "{{ ml is defined | bool }}" common__include_dw: "{{ dw is defined | bool }}" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 8cce89e2..55c2a1d8 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -65,8 +65,17 @@ run__ml_remove_storage: "{{ ml.remove_storage | default (run__force_ run__ml_public_loadbalancer: "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}" run__dw_definitions: "{{ dw.definitions | default([{}]) }}" -run__dw_suffix: "{{ dw.suffix | default('dw') }}" +run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" +run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" +run__dw_tags: "{{ dw.tags | default(common__tags) }}" +run__dw_overlay_network: "{{ dw.overlay_network | bool }}" +run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" +run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" +run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.default_dbc_suffix | default('-dl-default') }}" +run__dw_teardown: "{{ dw.teardown | default(omit) }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index e0bc0ade..6dca5c64 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -161,3 +161,44 @@ loop_control: loop_var: __ml_config label: "{{ config.name }}" + +- name: Prepare for CDP DW experiences + when: run__include_dw + block: + - name: Construct CDP DW Data catalog configurations + ansible.builtin.set_fact: + run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" + vars: + include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" + config: + name: "{{ __dw_config.name | default('') }}" + load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" + use_default_dbc: "{{ __dw_config.name is undefined | bool }}" + virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" + loop: "{{ run__dw_definitions }}" + loop_control: + loop_var: __dw_config + index_var: __dw_config_index + + - name: Construct CDP DW Virtual warehouse configurations + ansible.builtin.set_fact: + run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" + vars: + config: + name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + dbc_name: "{{ __dw_config.0.name }}" + use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" + type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" + template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" + autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}" + autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}" + common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}" + application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}" + ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}" + enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}" + tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" + loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" + loop_control: + loop_var: __dw_config + index_var: __dw_dbc_index + label: "{{ __dw_config.0.name }}" \ No newline at end of file diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml index 69d417a2..d5519c8a 100644 --- a/roles/runtime/tasks/initialize_setup_aws.yml +++ b/roles/runtime/tasks/initialize_setup_aws.yml @@ -15,7 +15,7 @@ # limitations under the License. - name: Discover AWS Public and Private VPC Subnets - when: not infra__aws_subnet_ids + when: infra__aws_subnet_ids is undefined block: - name: Query AWS Public Subnets amazon.aws.ec2_vpc_subnet_info: @@ -49,13 +49,17 @@ label: "{{ __aws_private_subnet_item.subnet_id }}" loop: "{{ __aws_private_subnets_info.subnets }}" + - name: Set default AWS Private Subnet if there are no Private Subnets + ansible.builtin.set_fact: + run__datahub_private_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) }}" + - name: Set fact for AWS Subnet IDs when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined ansible.builtin.set_fact: - run__datahub_subnet_ids: "{{ run__datahub_public_subnet_ids | default([]) | union(run__datahub_private_subnet_ids) }}" + run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | union(run__datahub_public_subnet_ids) }}" - name: Set fact for AWS Subnet IDs by assignment - when: infra__aws_subnet_ids + when: infra__aws_subnet_ids is defined block: - name: Set fact for All AWS Subnet IDs by assignment ansible.builtin.set_fact: diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index b798bfa8..ec6b7cdf 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -68,4 +68,111 @@ loop: "{{ run__env_info.environments[0].descendants.ml }}" loop_control: loop_var: __ml_config - label: "{{ __ml_config.instanceName }}" \ No newline at end of file + label: "{{ __ml_config.instanceName }}" + +- name: Discover CDW deployments + when: run__include_dw + block: + - name: Retrieve CDP DW experiences + cloudera.cloud.dw_cluster_info: + env: "{{ run__env_name }}" + register: __dw_list + + - name: Initialize CDP DW cluster id + ansible.builtin.set_fact: + run__dw_cluster_id: "{{ __dw_list.clusters[0].id }}" + + - name: Prepare partial teardown for CDW resources + when: run__dw_teardown is defined + block: + - name: Initialize list of CDW database catalogs to be decommissioned + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ [] }}" + + - name: Initialize list of CDW virtual warehouses to be decommissioned + ansible.builtin.set_fact: + run__dw_vw_ids: "{{ [] }}" + + - name: Populate CDW database catalogs defined with its id to the teardown list + when: __dbc.id is defined + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc.id ] }}" + loop: "{{ run__dw_teardown.database_catalogs }}" + loop_control: + loop_var: __dbc + label: "{{ __dbc.id }}" + + - name: Fetch the ids of all the CDW database catalogs defined with its name + when: __dbc.name is defined + cloudera.cloud.dw_database_catalog_info: + name: "{{ __dbc.name }}" + cluster_id: "{{ run__dw_cluster_id }}" + loop: "{{ run__dw_teardown.database_catalogs }}" + loop_control: + loop_var: __dbc + label: "{{ __dbc.name }}" + register: __dbc_name_list + + - name: Populate CDW database catalogs defined with its name to the teardown list + when: + - __dbc_result.database_catalogs is defined + - __dbc_result.database_catalogs | length > 0 + - __dbc_result.database_catalogs[0].id not in run__dw_dbc_ids + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ run__dw_dbc_ids + [ __dbc_result.database_catalogs[0].id ] }}" + loop: "{{ __dbc_name_list.results }}" + loop_control: + loop_var: __dbc_result + label: "{{ __dbc_result.database_catalogs[0].id }}" + + - name: Fetch all the CDW virtual warehouses under a CDW cluster + cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: "{{ run__dw_cluster_id }}" + register: __all_vw_list + + - name: Populate all the CDW virtual warehouses of database catalogs scheduled for teardown to the teardown list + when: + - run__dw_dbc_ids is defined + - __vw.dbcId in run__dw_dbc_ids + ansible.builtin.set_fact: + run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ] }}" + loop: "{{ __all_vw_list.virtual_warehouses }}" + loop_control: + loop_var: __vw + label: "{{ __vw.id }}" + + - name: Fetch the ids of all the CDW virtual warehouses defined with its name + when: + - run__dw_teardown.virtual_warehouses is defined + - __vw.name is defined + cloudera.cloud.dw_virtual_warehouse_info: + name: "{{ __vw.name }}" + cluster_id: "{{ run__dw_cluster_id }}" + loop: "{{ run__dw_teardown.virtual_warehouses }}" + loop_control: + loop_var: __vw + label: "{{ __vw.name }}" + register: __vw_name_list + + - name: Populate CDW virtual warehouses defined with its id to the teardown list + when: + - __vw.id is defined + - __vw.id not in run__dw_vw_ids + ansible.builtin.set_fact: + run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw.id ]}}" + loop: "{{ run__dw_teardown.virtual_warehouses }}" + loop_control: + loop_var: __vw + label: "{{ __vw.id }}" + + - name: Populate CDW virtual warehouses defined with its name to the teardown list + when: + - __vw_result.virtual_warehouses is defined + - __vw_result.virtual_warehouses | length > 0 + - __vw_result.virtual_warehouses[0].id not in run__dw_vw_ids + ansible.builtin.set_fact: + run__dw_vw_ids: "{{ run__dw_vw_ids + [ __vw_result.virtual_warehouses[0].id ]}}" + loop: "{{ __vw_name_list.results }}" + loop_control: + loop_var: __vw_result + label: "{{ __vw_result.virtual_warehouses[0].id }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 2c2132d4..ccd17f11 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -40,10 +40,10 @@ - name: Execute CDP DW cluster setup cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" - overlay: no - # TODO - Allow direct assignment (will need to coordinate with infra role) + overlay: "{{ run__dw_overlay_network }}" + private_load_balancer: "{{ run__dw_private_load_balancer }}" aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present wait: yes async: 3600 # 1 hour timeout @@ -60,4 +60,95 @@ register: __dw_builds_async until: __dw_builds_async.finished retries: 120 - delay: 30 \ No newline at end of file + delay: 30 + + - name: Retrieve CDP DW experiences + cloudera.cloud.dw_cluster_info: + env: "{{ run__env_name }}" + register: run__dw_list + + - name: Fetch all the Database Catalogs under the cluster + cloudera.cloud.dw_database_catalog_info: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + register: __dbc_list + + - name: Set default Database catalog id for the cluster + when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$') + ansible.builtin.set_fact: + __default_dbc_id: "{{ __dw_dbc.id }}" + loop: "{{ __dbc_list.database_catalogs }}" + loop_control: + loop_var: __dw_dbc + + - name: Create CDP DW Database catalogs + when: not __dw_dbc_config.use_default_dbc + cloudera.cloud.dw_database_catalog: + cluster_id : "{{ run__dw_list.clusters[0].id }}" + name: "{{ __dw_dbc_config.name }}" + load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" + state: present + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + loop: "{{ run__dw_dbc_configs }}" + loop_control: + loop_var: __dw_dbc_config + label: "{{ __dw_dbc_config.name }}" + register: __dw_dbc_builds + + - name: Wait for CDP DW Database catalogs setup to complete + when: __dw_dbc_build.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __dw_dbc_build.ansible_job_id }}" + register: __dw_dbc_builds_async + until: __dw_dbc_builds_async.finished + retries: 120 + delay: 30 + loop: "{{ __dw_dbc_builds.results }}" + loop_control: + loop_var: __dw_dbc_build + label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" + + - name: Set CDP DW Database catalog name to id map + when: __dw_dbc_build_async.database_catalog is defined + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}" + loop: "{{ __dw_dbc_builds_async.results }}" + loop_control: + loop_var: __dw_dbc_build_async + + - name: Create CDP DW Virtual warehouse + cloudera.cloud.dw_virtual_warehouse: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}" + type: "{{ __dw_vw_config.type }}" + name: "{{ __dw_vw_config.name }}" + template: "{{ __dw_vw_config.template }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}" + common_configs: "{{ __dw_vw_config.common_configs }}" + application_configs: "{{ __dw_vw_config.application_configs }}" + ldap_groups: "{{ __dw_vw_config.ldap_groups }}" + enable_sso: "{{ __dw_vw_config.enable_sso | bool }}" + tags: "{{ __dw_vw_config.tags }}" + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_vw_builds + loop: "{{ run__dw_vw_configs }}" + loop_control: + loop_var: __dw_vw_config + label: "{{ __dw_vw_config.name }}" + + - name: Wait for CDP DW Virtual warehouse setup to complete + when: __dw_vw_build.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __dw_vw_build.ansible_job_id }}" + register: __dw_vw_builds_async + until: __dw_vw_builds_async.finished + retries: 120 + delay: 30 + loop: "{{ __dw_vw_builds.results }}" + loop_control: + loop_var: __dw_vw_build + label: "{{ __dw_vw_build.__dw_vw_config.name }}" \ No newline at end of file diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index a5707698..8beeb2ec 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -32,17 +32,33 @@ poll: 0 register: __opdb_teardowns_info -- name: Execute CDP DW cluster teardown - register: __dw_teardown_info - when: - - run__include_dw or run__force_teardown | bool - - run__env_info.environments | length > 0 - - run__env_info.environments[0].descendants.dw | length > 0 +- name: Execute CDP DW teardown + when: run__dw_teardown is not defined cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent - wait: no + wait: yes force: "{{ run__dw_force_delete }}" + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_cluster_teardown_info + +- name: Execute CDP virtual warehouse partial teardown + when: + - run__dw_teardown is defined + - run__dw_vw_ids is defined + - run__dw_vw_ids | length > 0 + cloudera.cloud.dw_virtual_warehouse: + cluster_id: "{{ run__dw_cluster_id }}" + id: "{{ __vw_id }}" + state: absent + wait: yes + loop: "{{ run__dw_vw_ids }}" + loop_control: + loop_var: __vw_id + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_vw_teardown_info - name: Execute CDP Dataflow teardown register: __df_teardown_info @@ -129,15 +145,58 @@ retries: 120 delay: 30 -- name: Wait for CDP DW deployments to decommission +- name: Wait for CDP DW virtual warehouse to decommission when: - - __dw_teardown_info is defined - - __dw_teardown_info.started | default(False) - cloudera.cloud.dw_cluster: - env: "{{ run__env_name }}" - state: absent - wait: yes - force: "{{ run__dw_force_delete }}" + - __dw_vw_teardown_info is defined + - __dw_vw_teardown_info.results is defined + - __dw_vw_teardown_info.results | length > 0 + ansible.builtin.async_status: + jid: "{{ __dw_vw_teardown.ansible_job_id }}" + loop_control: + loop_var: __dw_vw_teardown + loop: "{{ __dw_vw_teardown_info.results }}" + register: __dw_vw_teardown_async + until: __dw_vw_teardown_async.finished + retries: 120 + delay: 30 + +- name: Execute CDP DW Database Catalogs partial teardown + when: run__dw_teardown.database_catalogs is defined + cloudera.cloud.dw_database_catalog: + cluster_id: "{{ run__dw_cluster_id }}" + id: "{{ __dbc_id }}" + state: absent + wait: yes + loop: "{{ run__dw_dbc_ids }}" + loop_control: + loop_var: __dbc_id + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_dbc_teardown_info + +- name: Wait for CDW Database Catalog to decommision + when: + - __dw_dbc_teardown_info is defined + - __dw_dbc_teardown_info.results is defined + - __dw_dbc_teardown_info.results | length > 0 + ansible.builtin.async_status: + jid: "{{ __dw_dbc_teardown.ansible_job_id }}" + loop_control: + loop_var: __dw_dbc_teardown + loop: "{{ __dw_dbc_teardown_info.results }}" + register: __dw_dbc_teardown_async + until: __dw_dbc_teardown_async.finished + retries: 120 + delay: 30 + +- name: Wait for CDW cluster to decommision + when: __dw_cluster_teardown_info.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __dw_cluster_teardown_info.ansible_job_id }}" + register: __dw_cluster_teardown_async + until: __dw_cluster_teardown_async.finished + retries: 120 + delay: 30 - name: Wait for CDP OpDB deployments to decommission when: diff --git a/roles/sequence/defaults/main.yml b/roles/sequence/defaults/main.yml index 598301db..7fc8f3b2 100644 --- a/roles/sequence/defaults/main.yml +++ b/roles/sequence/defaults/main.yml @@ -12,10 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -sequence__setup_runtime: "{{ common__setup_runtime }}" +sequence__setup_runtime: "{{ common__setup_runtime }}" -sequence__setup_plat: "{{ common__setup_plat }}" +sequence__setup_plat: "{{ common__setup_plat }}" -sequence__setup_infra: "{{ common__setup_infra }}" +sequence__setup_infra: "{{ common__setup_infra }}" -sequence_init: "{{ sequence__setup_infra }}" \ No newline at end of file +sequence_init: "{{ sequence__setup_infra }}" + +sequence__teardown_runtime: "{{ common__teardown_runtime }}" + +sequence__teardown_plat: "{{ common__teardown_plat }}" + +sequence__teardown_infra: "{{ common__teardown_infra }}" \ No newline at end of file diff --git a/roles/sequence/tasks/main.yml b/roles/sequence/tasks/main.yml index b77214fe..45b7ecff 100644 --- a/roles/sequence/tasks/main.yml +++ b/roles/sequence/tasks/main.yml @@ -59,6 +59,7 @@ - run - name: Initialize Runtime Teardown + when: sequence__teardown_runtime | bool ansible.builtin.include_role: name: cloudera.exe.runtime tasks_from: initialize_teardown @@ -74,6 +75,7 @@ - never - name: Teardown Runtime + when: sequence__teardown_runtime | bool ansible.builtin.include_role: name: cloudera.exe.runtime tasks_from: teardown @@ -89,6 +91,7 @@ - never - name: Initialize Platform Teardown + when: sequence__teardown_plat | bool ansible.builtin.include_role: name: cloudera.exe.platform tasks_from: initialize_teardown @@ -102,6 +105,7 @@ - never - name: Teardown Platform + when: sequence__teardown_plat | bool ansible.builtin.include_role: name: cloudera.exe.platform tasks_from: teardown @@ -115,6 +119,7 @@ - never - name: Initialize Infrastructure Teardown + when: sequence__teardown_infra | bool ansible.builtin.include_role: name: cloudera.exe.infrastructure tasks_from: initialize_teardown @@ -126,6 +131,7 @@ - never - name: Teardown Infrastructure + when: sequence__teardown_infra | bool ansible.builtin.include_role: name: cloudera.exe.infrastructure tasks_from: teardown