Add support for CDW Virtual Warehouses (#102)

cmperro · raju-saravanan · wmudge · web-flow · commit b71ddedeb975 · 2022-11-15T11:31:03.000-05:00
* Support CDW VW changes

Co-Authored-by: Saravanan Raju &lt;saravanan.footloose@gmail.com&gt;
Co-Authored-by: Webster Mudge &lt;wmudge@cloudera.com&gt;
Co-Authored-by: Chris Perro &lt;cmperro@gmail.com&gt;
Signed-off-by: Chris Perro &lt;cmperro@gmail.com&gt;
diff --git a/docs/configuration.yml b/docs/configuration.yml
@@ -30,6 +30,7 @@ datahub:
     gcp:
   suffix:
   tags:
+  force_delete:
 de:
   definitions:
   suffix:
@@ -82,7 +83,35 @@ df:
                   id: SECONDS|MINUTES|HOURS|DAYS
 dw:
   definitions:
+    - name:
+      use_default_dbc:
+      load_demo_data:
+      virtual_warehouses:
+        - name:
+          type:
+          template:
+          autoscaling:
+            min_nodes:
+            max_nodes:
+          tags:
+          configs:
+            common_configs:
+            application_configs:
+            enable_sso:
+            ldap_groups:
   suffix:
+  vw:
+    suffix:
+    type:
+    template:
+  dbc:
+    suffix:
+    default_suffix: 
+  tags:
+  overlay_network:
+  private_load_balancer:
+  private_worker_nodes:
+  force_delete:
 env:
   aws:
     arn_partition: aws | aws-cn | aws-us-gov (See https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html)
@@ -250,6 +279,7 @@ globals:
     vm:
       count:
       os:
+  force_teardown:
   gcloud_credential_file:
   infra_deployment_engine:
   infra_type:
@@ -413,6 +443,7 @@ ml:
   suffix:
   tags:
   public_loadbalancer:
+  force_delete:
 opdb:
   definitions:
   suffix:
diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml
@@ -111,6 +111,7 @@ common__aws_public_subnet_ids:            "{{ infra.aws.vpc.existing.public_subn
 common__aws_private_subnet_ids:           "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
 common__aws_region:                       "{{ infra.aws.region | default('eu-west-1') }}"
 common__aws_role_suffix:                  "{{ infra.aws.role.suffix | default(common__role_suffix) }}"
+
 common__aws_datalake_admin_role_name:     "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}"
 common__aws_datalake_admin_suffix:        "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}"
 common__aws_idbroker_role_name:           "{{ env.aws.role.name.idbroker | default([common__namespace, common__aws_idbroker_suffix, common__aws_role_suffix] | join('-')) }}"
diff --git a/roles/common/meta/main.yml b/roles/common/meta/main.yml
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 galaxy_info:
+  role_name: platform
+  namespace: cloudera
   author: Webster Mudge (wmudge@cloudera.com)
   description: >
     Shared configuration variables managed by role dependency. 
diff --git a/roles/infrastructure/defaults/main.yml b/roles/infrastructure/defaults/main.yml
@@ -95,9 +95,9 @@ infra__vpc_user_ports:              "{{ infra.vpc.user_ports | default([infra__a
 infra__vpc_user_cidr:               "{{ infra.vpc.user_cidr | default([]) }}"
 infra__vpc_tunneled_cidr:           "{{ infra.vpc.tunneled_cidr | default([]) }}"
 
-infra__aws_vpc_id:                  "{{ infra.aws.vpc.existing.vpc_id | default('') }}"
-infra__aws_public_subnet_ids:       "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}"
-infra__aws_private_subnet_ids:      "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
+infra__aws_vpc_id:                  "{{ common__aws_vpc_id }}"
+infra__aws_public_subnet_ids:       "{{ common__aws_public_subnet_ids }}"
+infra__aws_private_subnet_ids:      "{{ common__aws_private_subnet_ids }}"
 
 infra__security_group_knox_name:    "{{ common__security_group_knox_name }}"
 infra__security_group_default_name: "{{ common__security_group_default_name }}"
diff --git a/roles/infrastructure/tasks/initialize_aws.yml b/roles/infrastructure/tasks/initialize_aws.yml
@@ -115,7 +115,6 @@
 
         - name: Set facts for existing AWS Public Subnet IDs
           ansible.builtin.set_fact:
-            infra__aws_public_subnet_ids: "{{ infra__aws_public_subnet_ids }}"
             infra__aws_subnet_ids: "{{ infra__aws_subnet_ids | default([]) | union(infra__aws_public_subnet_ids) }}"
             infra__aws_vpc_id: "{{ __aws_public_subnets_info.subnets | map(attribute='vpc_id') | list | first }}"
 
diff --git a/roles/infrastructure/tasks/setup.yml b/roles/infrastructure/tasks/setup.yml
@@ -33,7 +33,7 @@
     ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_utility_service.yml"
 
   - name: Set up provider-specific Infrastructure Compute
-    when: infra__dynamic_inventory_count
+    when: infra__dynamic_inventory_count | int > 0
     ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_compute.yml"
 
 - name: Set up for Terraform deployment engine
diff --git a/roles/platform/defaults/main.yml b/roles/platform/defaults/main.yml
@@ -120,6 +120,9 @@ plat__aws_storage_suffix:                     "{{ env.aws.storage.suffix | defau
 plat__aws_role_tags:                          "{{ env.aws.role.tags | default({}) }}"
 plat__aws_policy_tags:                        "{{ env.aws.policy.tags | default({}) }}"
 plat__aws_storage_tags:                       "{{ env.aws.storage.tags | default({}) }}"
+plat__aws_vpc_id:                             "{{ common__aws_vpc_id }}"
+plat__aws_public_subnet_ids:                  "{{ common__aws_public_subnet_ids }}"
+plat__aws_private_subnet_ids:                 "{{ common__aws_private_subnet_ids }}"
 
 plat__aws_xaccount_suffix:                    "{{ env.aws.role.label.cross_account | default(common__xaccount_suffix) }}"
 plat__aws_idbroker_suffix:                    "{{ common__aws_idbroker_suffix }}"
diff --git a/roles/platform/meta/main.yml b/roles/platform/meta/main.yml
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 galaxy_info:
+  role_name: platform
+  namespace: cloudera
   author: Webster Mudge (wmudge@cloudera.com)
   description: >
     Deployment and management of Cloudera Data Platform (CDP) Public Cloud core
diff --git a/roles/platform/tasks/initialize_aws.yml b/roles/platform/tasks/initialize_aws.yml
@@ -17,4 +17,9 @@
 - name: Retrieve AWS Caller details
   amazon.aws.aws_caller_info:
   register: __aws_caller_info
-  failed_when: __aws_caller_info.account is not defined
+  failed_when: __aws_caller_info.account is not defined
+
+- name: Confirm AWS SSH Public Key ID exists
+  ansible.builtin.command: aws ec2 describe-key-pairs --region "{{ plat__region }}" --key-name "{{ plat__public_key_id }}"
+  register: __aws_ssh_key_pair
+  failed_when: __aws_ssh_key_pair.rc != 0
diff --git a/roles/platform/tasks/initialize_setup_aws.yml b/roles/platform/tasks/initialize_setup_aws.yml
@@ -49,7 +49,7 @@
     plat__aws_xaccount_account_id: "{{ plat__cdp_xaccount_account_id }}"
 
 # Runlevel first, upstream second, and discover third
-- name: Discover AWS VPC if not defined
+- name: Discover AWS VPC if not defined or established by Infrastructure
   when: plat__aws_vpc_id == "" and infra__aws_vpc_id is undefined
   block:
     - name: Query AWS VPC by name
@@ -71,13 +71,12 @@
         plat__aws_vpc_id: "{{ __aws_vpc_info.vpcs[0].id }}"
 
 - name: Set fact for AWS VPC ID if established by Infrastructure
-  when: infra__aws_vpc_id is defined
+  when: plat__aws_vpc_id == "" and infra__aws_vpc_id is defined
   ansible.builtin.set_fact:
     plat__aws_vpc_id: "{{ infra__aws_vpc_id }}"
 
-# Runlevel first, upstream second, and discover third
-- name: Handle AWS Public and Private VPC Subnets if not defined
-  when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids
+- name: Handle AWS Subnet IDs if not defined
+  when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids # Defaults are empty lists
   block:
     - name: Query AWS Subnets
       amazon.aws.ec2_vpc_subnet_info:
@@ -144,13 +143,15 @@
   ansible.builtin.set_fact:
     plat__endpoint_access_scheme: "PUBLIC"
 
+# TODO Collapse the two SG queries together
 - name: Discover AWS Security Group for Knox
   when: infra__aws_security_group_knox_id is undefined
   block:
     - name: Query AWS Security Group for Knox
       amazon.aws.ec2_group_info:
         region: "{{ plat__region }}"
         filters:
+          vpc-id: "{{ plat__aws_vpc_id }}"
           group-name: "{{ plat__security_group_knox_name }}"
       register: __aws_security_group_knox_info
 
@@ -171,6 +172,7 @@
       amazon.aws.ec2_group_info:
         region: "{{ plat__region }}"
         filters:
+          vpc-id: "{{ plat__aws_vpc_id }}"
           group-name: "{{ plat__security_group_default_name }}"
       register: __aws_security_group_default_info
 
diff --git a/roles/platform/tasks/setup_aws_env.yml b/roles/platform/tasks/setup_aws_env.yml
@@ -31,7 +31,7 @@
     subnet_ids: "{{ plat__aws_public_subnet_ids | union(plat__aws_private_subnet_ids) }}"
     tags: "{{ plat__tags }}"
     tunnel: "{{ plat__tunnel }}"
-    endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}"
+    endpoint_access_scheme: "{{ plat__public_endpoint_access | ternary('PUBLIC', omit) }}"
     endpoint_access_subnets: "{{ plat__aws_public_subnet_ids | default(omit) }}"
     freeipa:
       instanceCountByGroup: "{{ plat__env_freeipa }}"
diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml
@@ -71,8 +71,17 @@ run__de_force_delete:               "{{ de.force_delete | default (run__force_te
 run__de_vc_suffix:                  "{{ de.vc.suffix | default('vc') }}"
 
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
-run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
+run__dw_dbc_suffix:                 "{{ dw.dbc.suffix | default('dbc') }}"
+run__dw_vw_suffix:                  "{{ dw.vw.suffix | default('vw') }}"
+run__dw_tags:                       "{{ dw.tags | default(common__tags) }}"
+run__dw_overlay_network:            "{{ dw.overlay_network | default(False) | bool }}"
+run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
+run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | default(False) | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
+run__dw_default_vw_type:            "{{ dw.default_vw.type | default('hive') }}"
+run__dw_default_template_type:      "{{ dw.default_template.type | default('xsmall') }}"
+run__dw_default_dbc_suffix:         "{{ dw.default_dbc.suffix | default('dl-default') }}"
+run__dw_default_dbc:                "{{ dw.default_dbc.name | default([run__env_name, run__dw_default_dbc_suffix] | join('-')) }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"
diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml
@@ -52,6 +52,7 @@
       ansible.builtin.set_fact:
         run__cdp_datalake_version: "{{ __cdp_datalake_version_info.versions[0].runtimeVersion | trim }}"
 
+# TODO Discover version if upstream is not present
 - name: Set fact for CDP Datalake version by assignment
   when: plat__cdp_datalake_version is defined
   ansible.builtin.set_fact:
@@ -64,7 +65,7 @@
     - name: Retrieve Image Catalog File
       ansible.builtin.uri:
         url: "{{ run__datahub_image_catalog_url }}"
-      #no_log: yes
+      no_log: yes
       register: __datahub_image_catalog
 
     - name: Set fact for latest CDP Image in Catalog
@@ -184,6 +185,59 @@
         loop_var: __de_config
         label: "{{ config.name }}"
 
+
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
+  block:
+    - name: Construct CDP DW Data Catalog configurations
+      ansible.builtin.set_fact:
+        run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        config:
+          name: "{{ __dw_config.name | default(run__dw_default_dbc) }}"
+          load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
+          virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
+          use_default_dbc: "{{ __dw_config.use_default_dbc | default(True) | bool }}"
+      loop: "{{ run__dw_definitions }}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_config_index
+
+    - name: Construct CDP DW Virtual Warehouse configurations
+      ansible.builtin.set_fact:
+        run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
+      vars:
+        config:
+          dbc_name: "{{ __dw_config.0.name }}"
+          name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          use_default_dbc: "{{ __dw_config.0.use_default_dbc }}"
+          type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}"
+          template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}"
+          tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}"
+          autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}"
+          configs: "{{ __dw_config.1.configs | default({}) }}"
+      loop: "{{ run__dw_dbc_configs | default({}) | subelements('virtual_warehouses')}}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_dbc_index
+        label: "{{ config.name }}"
+
+    - name: Check CDP DW Virtual Warehouse tags
+      when: __dw_vw_config.tags | length > 0
+      ansible.builtin.assert:
+        that:
+          - __dw_vw_config.tags | dict2items | rejectattr('value', 'regex', '[^-_a-zA-Z0-9.=:+@]+') | list
+        fail_msg: 
+          - "A tag in Data Warehouse, '{{ __dw_vw_config.name }}', does not meet requirements;"
+          - "current tags: {{ __dw_vw_config.tags}}."
+          - "Allowed characters in tags are letters, numbers and the following characters: _.:/=+-@"
+        quiet: yes
+      loop_control:
+        loop_var: __dw_vw_config
+        label: "{{ __dw_vw_config.name }}"
+      loop: "{{ run__dw_vw_configs }}"
+
 - name: Prepare for CDP DF Service experiences
   tags: df
   when:
diff --git a/roles/runtime/tasks/initialize_setup.yml b/roles/runtime/tasks/initialize_setup.yml
@@ -31,3 +31,19 @@
     - dh
     - df
     - de
+
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
+  block:
+    - name: Confirm public subnet count CDP DW public load balancer
+      when: not run__dw_private_load_balancer
+      ansible.builtin.assert:
+        that:
+          - run__public_subnet_ids | length == 3
+        fail_msg: "Must have exactly 3 public subnets when deploying CDP Data Warehouse with a public load balancer"
+        quiet: yes
+      tags:
+        - ml
+        - dw
+        - opdb
+        - dh
diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml
@@ -24,11 +24,38 @@
   when: not run__force_teardown
   ansible.builtin.include_tasks: "initialize_base.yml"
 
-- name: Discover CDP DF Deployments
-  register: run__df_service_info
+- name: Discover CDP Dataflow deployments
   when: run__include_df
   cloudera.cloud.df_service_info:
     name: "{{ run__env_name }}"
+  register: run__df_env_info
+
+- name: Discover CDP Data Warehouse cluster
+  cloudera.cloud.dw_cluster_info:
+    env: "{{ run__env_name }}"
+  register: __dw_list
+
+- name: Discover CDP Data Warehouse deployments
+  when: 
+    - run__include_dw
+    - not run__force_teardown | bool or not run__dw_force_delete | bool
+    - __dw_list.clusters | length > 0
+  block:
+    - name: Initialize CDP Data Warehouse cluster id
+      ansible.builtin.set_fact:
+        __dw_cluster_id: "{{ __dw_list.clusters | map(attribute='id') | first | default(omit) }}"
+
+    - name: Discover CDP Data Warehouse database catalogs
+      when: __dw_cluster_id is defined
+      cloudera.cloud.dw_database_catalog_info:
+        cluster_id: "{{ __dw_cluster_id }}"
+      register: __dw_dbc_list
+
+    - name: Discover CDP Data Warehouse virtual warehouses
+      when: __dw_cluster_id is defined
+      cloudera.cloud.dw_virtual_warehouse_info:
+        cluster_id: "{{ __dw_cluster_id }}"
+      register: __dw_vw_list
 
 - name: Initialize Purge of all Runtimes in Environment
   when:
diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml
diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml
diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml
