Support Private Networks (#15)

* Add level-1 private networking to deploy tool.
* Support creation of multiple nat gateway and private route table.
* Place all the private network changes behind a flag
* Use env.tunnel instead of globals.env_network_type and fix syntactic issues
* Excluding CDP Cidr from inbound security group.
* Addressed the comments using existing properties and introduced publicendpoint access for AWS
* Add public endpoint gateway support for experiences
* Support level-1 network for CDW
* Avoids duplicate route table creation of one exists and fixes error handling in NAT gateway failures

Signed-off-by: guruchai <ggchaitanya@gmail.com>

Co-authored-by: Saravanan Raju <sraju@cloudera.com>
Co-authored-by: Saravanan Raju <saravanan.footloose@gmail.com>

Author: 3 people

docs/configuration.yml

@@ -169,7 +169,11 @@ env:
     delete_policies:
     delete_roles:
     delete_user_group:
+  # Setting tunnel option to true enables CCM gateway which removes the need for the environment hosts to have a public IP address
   tunnel:
+  # Setting public_endpoint_access to true enables public workload endpoint access gateway which lets user access workload from internet
+  # default is false. Needing when tunneling is enabled but dont have the direct connectivity with the VPC.
+  public_endpoint_access:
   workload_analytics:
 globals:
   admin_password:
@@ -205,7 +209,6 @@ globals:
     public:
     ranger_audit:
     role:
-    route_table:
     service_network:
     storage:
     subnet:
@@ -237,9 +240,10 @@ infra:
         name:
         suffix:
       labels:
-        route_table:
-      route_table:
-        suffix:
+        public_route_table:
+        private_route_table:
+        public_route_table_suffix:
+        private_route_table_suffix:
   azure:
     metagroup:
       name:
@@ -323,6 +327,7 @@ ml:
   k8s_request_base:
   suffix:
   tags:
+  public_loadbalancer:
 opdb:
   definitions:
   suffix:

roles/common/defaults/main.yml

@@ -45,11 +45,11 @@ common__datalake_admin_suffix:            "{{ globals.labels.datalake_admin | de
 common__ranger_audit_suffix:              "{{ globals.labels.ranger_audit | default('audit') }}"
 common__cml_suffix:                       "{{ globals.labels.cml | default('cml') }}"
 common__igw_suffix:                       "{{ globals.labels.internet_gateway | default('igw') }}"
-common__route_table_suffix:               "{{ globals.labels.route_table | default('rtb') }}"
 common__app_suffix:                       "{{ globals.labels.app | default('app') }}"
 common__group_suffix:                     "{{ globals.labels.group | default('group') }}"
 common__admin_suffix:                     "{{ globals.labels.admin | default('admin') }}"
 common__user_suffix:                      "{{ globals.labels.user | default('user') }}"
+common__ngw_suffix:                       "{{ globals.labels.nat_gateway | default('ngw') }}"
 
 common__unique_storage_name_suffix:       "{{ globals.storage.name | default((common__region + common__aws_profile) if 'aws' in common__infra_type else common__region) }}"
 
@@ -121,6 +121,8 @@ common__env_name_suffix:                  "{{ env.suffix | default(common__env_s
 
 common__datalake_name:                    "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}"
 common__datalake_name_suffix:             "{{ env.datalake.suffix | default(common__datalake_suffix) }}"
+common__tunnel:                           "{{ env.tunnel | default(False) }}"
+common__public_endpoint_access:           "{{ env.public_endpoint_access | default(not common__tunnel) }}"
 
 common__env_admin_password:                     "{{ globals.admin_password | mandatory }}"
 # Deploy

roles/infrastructure/defaults/main.yml

@@ -28,6 +28,7 @@ infra__vpc_public_subnets_suffix:   "{{ common__vpc_public_subnets_suffix }}"
 
 # Infra
 infra__type:                        "{{ common__infra_type }}"
+infra__tunnel:                      "{{ common__tunnel }}"
 
 # Dynamic Inventory for Clusters
 infra__private_key_file:                 "{{ globals.ssh.private_key_file | default('') }}"
@@ -90,8 +91,14 @@ infra__aws_dynamodb_table_name:     "{{ common__aws_dynamodb_table_name }}"
 infra__aws_vpc_az_count:            "{{ infra.aws.vpc.az_count | default(3) }}"
 infra__aws_igw_name:                "{{ infra.aws.vpc.internet_gateway.name | default([infra__namespace, infra__aws_igw_suffix] | join('-')) }}"
 infra__aws_igw_suffix:              "{{ infra.aws.vpc.internet_gateway.suffix | default(common__igw_suffix) }}"
-infra__aws_route_table_name:        "{{ infra.aws.vpc.labels.route_table | default([infra__namespace, infra__aws_route_table_suffix] | join('-')) }}"
-infra__aws_route_table_suffix:      "{{ infra.aws.vpc.route_table.suffix | default(common__route_table_suffix) }}"
+
+infra__aws_public_route_table_suffix:  "{{ infra.aws.vpc.labels.public_route_table_suffix | default('public-rtb') }}"
+infra__aws_private_route_table_suffix: "{{ infra.aws.vpc.labels.private_route_table_suffix | default('private-rtb') }}"
+infra__aws_public_route_table_name: "{{ infra.aws.vpc.labels.public_route_table | default([infra__namespace, infra__aws_public_route_table_suffix] | join('-')) }}"
+infra__aws_private_route_table_name: "{{ infra.aws.vpc.labels.private_route_table | default([infra__namespace, infra__aws_private_route_table_suffix] | join('-')) }}"
+
+infra__aws_nat_gateway_name:        "{{ infra.aws.vpc.nat_gateway.name | default([infra__namespace, infra__aws_nat_gateway_suffix] | join('-')) }}"
+infra__aws_nat_gateway_suffix:      "{{ infra.aws.vpc.nat_gateway.suffix | default(common__ngw_suffix) }}"
 
 # GCP
 infra__gcp_project:                 "{{ common__gcp_project }}"

roles/infrastructure/tasks/initialize_setup_aws.yml

@@ -91,6 +91,7 @@
         cidr_ip: "{{ infra__vpc_extra_cidr }}"
 
 - name: Add CDP Public Cloud security group rules for AWS
+  when: not infra__tunnel
   ansible.builtin.set_fact:
     infra__aws_security_group_rules: "{{ infra__aws_security_group_rules | union([rule]) }}"
   vars:

roles/infrastructure/tasks/setup_aws_network.yml

@@ -40,29 +40,28 @@
   ansible.builtin.set_fact:
     infra__aws_igw_id: "{{ __aws_igw.gateway_id }}"
 
-- name: Create AWS VPC Subnets
+- name: Create AWS VPC Public Subnets
   amazon.aws.ec2_vpc_subnet:
     region: "{{ infra__region }}"
     vpc_id: "{{ infra__aws_vpc_id }}"
-    cidr: "{{ __aws_subnet_create_item.cidr }}"
+    cidr: "{{ __aws_public_subnet_item.cidr }}"
     state: present
-    tags: "{{ infra__tags | combine(__aws_subnet_create_item.tags, recursive=True) }}"
+    tags: "{{ infra__tags | combine(__aws_public_subnet_item.tags, recursive = true) }}"
     map_public: yes
     az: "{{ __aws_az_info.availability_zones[__aws_subnet_index % infra__aws_vpc_az_count | int].zone_name }}"
   loop_control:
-    loop_var: __aws_subnet_create_item
+    loop_var: __aws_public_subnet_item
     index_var: __aws_subnet_index
-    label: __aws_subnet_create_item.name
-  loop: "{{ infra__vpc_public_subnets_info | union(infra__vpc_private_subnets_info) }}"
-  register: __aws_subnets
+  loop: "{{ infra__vpc_public_subnets_info }}"
+  register: __aws_public_subnets
 
-- name: Set fact for AWS Subnet IDs
+- name: Set fact for AWS Public Subnet IDs
   ansible.builtin.set_fact:
-    infra__aws_subnet_ids: "{{ infra__aws_subnet_ids | default([]) | union([__aws_subnet_item.subnet.id | default('')]) }}"
+    infra__aws_public_subnet_ids: "{{ infra__aws_public_subnet_ids | default([]) | union([__aws_subnet_item.subnet.id | default('')]) }}"
   loop_control:
     loop_var: __aws_subnet_item
     label: "{{ __aws_subnet_item.subnet.id }}"
-  loop: "{{ __aws_subnets.results }}"
+  loop: "{{ __aws_public_subnets.results }}"
 
 - name: List the Route Tables for the VPC
   community.aws.ec2_vpc_route_table_info:
@@ -71,20 +70,84 @@
       vpc-id: "{{ infra__aws_vpc_id }}"
   register: __aws_route_table_list
 
-- name: Configure the Route Table
+- name: Configure the Public Route Table
   community.aws.ec2_vpc_route_table:
     region: "{{ infra__region }}"
     vpc_id: "{{ infra__aws_vpc_id }}"
     route_table_id: "{{ __aws_route_table_id }}"
     lookup: id
     state: present
-    tags: "{{ infra__tags | combine({ 'Name': infra__aws_route_table_name }, recursive=True) }}"
+    tags: "{{ infra__tags | combine({ 'Name': infra__aws_public_route_table_name }, recursive=True) }}"
     routes:
       - dest: "0.0.0.0/0"
         gateway_id: "{{ infra__aws_igw_id }}"
-    subnets: "{{ infra__aws_subnet_ids }}"
+    subnets: "{{ infra__aws_public_subnet_ids }}"
   vars:
-    __aws_route_table_id: "{{ __aws_route_table_list.route_tables[0].associations[0].route_table_id }}"
+    __aws_route_table_id: "{{ __aws_route_table_list.route_tables | json_query(__aws_rtb_jq) | flatten | first }}"
+    __aws_rtb_jq: "[*].associations[?main == `true` ].route_table_id"
+
+- name: Set the fact for Subnet Ids
+  when: not infra__tunnel
+  ansible.builtin.set_fact:
+    infra__aws_subnet_ids: "{{ infra__aws_public_subnet_ids }}"
+
+- name: Setup for private networking
+  when: infra__tunnel
+  block:
+    - name: Create AWS VPC Private Subnets
+      amazon.aws.ec2_vpc_subnet:
+        region: "{{ infra__region }}"
+        vpc_id: "{{ infra__aws_vpc_id }}"
+        cidr: "{{ __aws_private_subnet_item.cidr }}"
+        state: present
+        tags: "{{ infra__tags | combine(__aws_private_subnet_item.tags, recursive = true) }}"
+        map_public: no
+        az: "{{ __aws_az_info.availability_zones[__aws_subnet_index % infra__aws_vpc_az_count | int].zone_name }}"
+      loop_control:
+        loop_var: __aws_private_subnet_item
+        index_var: __aws_subnet_index
+        label: "{{ __aws_private_subnet_item.name }}"
+      loop: "{{ infra__vpc_private_subnets_info }}"
+      register: __aws_private_subnets
+
+    - name: Set fact for AWS Private Subnet IDs
+      ansible.builtin.set_fact:
+        infra__aws_private_subnet_ids: "{{ infra__aws_private_subnet_ids | default([]) | union([__aws_subnet_item.subnet.id | default('')]) }}"
+      loop_control:
+        loop_var: __aws_subnet_item
+      loop: "{{ __aws_private_subnets.results }}"
+
+    - name: Set fact for Subnet Ids
+      ansible.builtin.set_fact:
+        infra__aws_subnet_ids: "{{ infra__aws_public_subnet_ids | union(infra__aws_private_subnet_ids) }}"
+
+    - name: Creates NAT gateways and allocates EIPs
+      community.aws.ec2_vpc_nat_gateway:
+        state: present
+        subnet_id: "{{ __aws_public_subnet_id }}"
+        wait: true
+        if_exist_do_not_create: true
+        region: "{{ infra__region }}"
+        tags: "{{ infra__tags | combine({ 'Name': '-'.join([infra__aws_nat_gateway_name, __aws_public_subnet_index | string]) }, recursive=True) }}"
+      loop_control:
+        loop_var: __aws_public_subnet_id
+        index_var: __aws_public_subnet_index
+      loop: "{{ infra__aws_public_subnet_ids }}"
+      register: __aws_ngws
+
+    - name: Configure Private Route Tables
+      community.aws.ec2_vpc_route_table:
+        vpc_id: "{{ infra__aws_vpc_id }}"
+        region: "{{ infra__region }}"
+        tags: "{{ infra__tags | combine({ 'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string]) }, recursive=True) }}"
+        subnets: "{{ __aws_private_subnet_id_item }}"
+        routes:
+        - dest: "0.0.0.0/0"
+          nat_gateway_id:  "{{ __aws_ngws.results[ __aws_private_subnet_id_index % __aws_ngws.results | length ].nat_gateway_id }}"
+      loop_control:
+        loop_var: __aws_private_subnet_id_item
+        index_var: __aws_private_subnet_id_index
+      loop: "{{ infra__aws_private_subnet_ids }}"
 
 - name: Create Security Groups
   amazon.aws.ec2_group:

roles/infrastructure/tasks/teardown_aws_network.yml

@@ -29,6 +29,47 @@
         - "{{ infra__security_group_knox_name }}"
         - "{{ infra__security_group_default_name }}"
 
+    - name: Remove the private networking setup
+      when: infra__tunnel
+      block:
+        - name: Delete the private route tables
+          community.aws.ec2_vpc_route_table:
+            vpc_id: "{{ infra__aws_vpc_id }}"
+            region: "{{ infra__region }}"
+            lookup: tag
+            tags: "{{ { 'Name': '-'.join([infra__aws_private_route_table_name, __aws_private_subnet_id_index | string])} }}"
+            state: absent
+          loop_control:
+            index_var: __aws_private_subnet_id_index
+          loop: "{{ infra__vpc_private_subnet_cidrs }}"
+
+        - name: List all managed nat gateways within this VPC
+          community.aws.ec2_vpc_nat_gateway_info:
+            region: "{{ infra__region }}"
+            filters:
+              vpc-id: "{{ infra__aws_vpc_id }}"
+          register: __aws_all_ngws
+
+        - name: Delete nat gateway using discovered nat gateways from facts module.
+          community.aws.ec2_vpc_nat_gateway:
+            state: absent
+            region: "{{ infra__region }}"
+            wait: true
+            nat_gateway_id: "{{ item.nat_gateway_id }}"
+            release_eip: true
+          register: __aws_ngw_teardown
+          loop_control:
+            label: "{{ item.nat_gateway_id }}"
+          loop: "{{ __aws_all_ngws.result }}"
+          ignore_errors: true
+
+        - name: Check if NAT gateways are deleted succesfully
+          when: __aws_ngw_teardown is defined and __aws_ngw_teardown.results is defined and __aws_ngw_teardown.results | count > 0
+          ansible.builtin.fail:
+            msg: "Failed to delete a NAT gateway"
+          failed_when: item.rc is defined and item.rc != 1 and ('InvalidAllocationID.NotFound' in item.module_stderr)
+          loop: "{{ __aws_ngw_teardown.results }}"
+
     - name: Remove VPC subnets
       amazon.aws.ec2_vpc_subnet:
         region: "{{ infra__region }}"

roles/platform/defaults/main.yml

@@ -61,7 +61,8 @@ plat__teardown_deletes_user_group:            "{{ env.teardown.delete_user_group
 plat__xacccount_credential_name:              "{{ common__xaccount_credential_name }}"
 
 plat__workload_analytics:                     "{{ env.workload_analytics | default(True) }}"
-plat__tunnel:                                 "{{ env.tunnel | default(True) }}"
+plat__tunnel:                                 "{{ common__tunnel }}"
+plat__public_endpoint_access:                 "{{ common__public_endpoint_access }}"
 
 plat__env_admin_password:                     "{{ common__env_admin_password }}"
 

roles/platform/tasks/initialize_setup_aws.yml

@@ -92,6 +92,12 @@
   ansible.builtin.set_fact:
     plat__aws_subnet_ids: "{{ infra__aws_subnet_ids }}"
 
+- name: Set public subnets for public endpoint access
+  when: plat__public_endpoint_access
+  ansible.builtin.set_fact:
+    plat__aws_public_subnet_ids: "{{ infra__aws_public_subnet_ids }}"
+    plat__endpoint_access_scheme: "PUBLIC"
+
 - name: Discover AWS Security Group for Knox
   when: infra__aws_security_group_knox_id is undefined
   block:

roles/platform/tasks/setup_aws_env.yml

@@ -31,3 +31,6 @@
     subnet_ids: "{{ plat__aws_subnet_ids }}"
     s3_guard_name: "{{ plat__aws_dynamodb_table_name }}"
     tags: "{{ plat__tags }}"
+    tunnel: "{{ plat__tunnel }}"
+    endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}"
+    endpoint_access_subnets: "{{ plat__aws_public_subnet_ids | default(omit) }}"

roles/platform/vars/main.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Vars for platform
-plat__aws_policy_urls_default_root: "https://github.com/hortonworks/cloudbreak/raw/master/cloud-aws-common/src/main/resources/definitions/cdp"
+plat__aws_policy_urls_default_root: "https://raw.githubusercontent.com/hortonworks/cloudbreak/master/cloud-aws-common/src/main/resources/definitions/cdp"
 plat__aws_policy_urls_default:
   log:                "{{ plat__aws_policy_urls_default_root }}/aws-cdp-log-policy.json"
   ranger_audit_s3:    "{{ plat__aws_policy_urls_default_root }}/aws-cdp-ranger-audit-s3-policy.json"