Improve teardown and support purge mode

* Improve fetching of env_crn in Runtime role to not rely on Platform role
* Have DF teardown only execute when an Environment is found
* Introduce 'purge' keyword for Definitions to signal that an AWS VPC and all child objects, and all unattached EBS volumes in the name_prefix, should be removed from the given infra_region
* Add network, compute, and storage discovery to AWS Initialization
* Add teardown of discovered network, compute and storage when purge is used
* Enforce check that a CDP Environment has no listed descendants before allowing teardown
* Add new Runtime purge discovery and execution controlled by new purge key
* Move Datahub purge to after Experiences that lean on Datahubs for execution to allow controlled removal to be tried first
* Improve async wait for Runtime removal to complete
* Add force delete, and force data removal options to Runtimes where appropriate
* Standardise usage of json_query to correctly use FQCN of community.general.json_query
* Add warning to Create AWS Buckets function for common error where user attempts to create a bucket that is already owned elsewhere
* Remove unused Runtime service info registrations
* Make EKS Teardown async in case of multiple orphaned clusters
* Handle edge case for network teardown where the entry exists but has no objects to remove
* Reorder AWS Network teardown so that NGW and RTB are removed before cleaning ENI that may be using them
* Improve setting CDP Environment CRN during Runtime role Discovery when an Environment matching the name is found. Do not default to an empty string.
* Add task to discover list of DF Service Deployments matching target Environment during Runtime role execution.
* Improve logic around when to run DF and DW teardown tasks.
* Add option to loop over DF Service teardown for situations where multiple Disabled services may need purging besides the single Active service.
* Source force_teardown from globals rather than from top level key

Signed-off-by: Daniel Chaffelson <chaffelson@gmail.com>

Author: Chaffelson

roles/common/defaults/main.yml

@@ -138,4 +138,7 @@ common__include_dw:                        "{{ dw is defined | bool }}"
 common__include_de:                        "{{ de is defined | bool }}"
 common__include_df:                        "{{ df is defined | bool }}"
 common__include_datahub:                   "{{ datahub is defined | bool }}"
-common__include_opdb:                      "{{ opdb is defined | bool }}"
+common__include_opdb:                      "{{ opdb is defined | bool }}"
+
+# Teardown
+common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution

roles/infrastructure/defaults/main.yml

@@ -133,3 +133,7 @@ infra__cdp_control_plane_cidr:      "{{ env.cdp.control_plane.cidr | default(inf
 # Utility Service for Download Mirror
 infra__create_utility_service:      "{{ globals.create_utility_service | default('no') | bool }}"
 infra__utlity_bucket_name:          "{{ globals.utility_bucket_name | default('') }}"
+
+# Teardown
+infra__force_teardown:              "{{ common__force_teardown }}"
+infra__env_name:                    "{{ common__env_name }}"  # Used for purge lookups

roles/infrastructure/tasks/initialize_aws.yml

@@ -88,4 +88,133 @@
 - name: Set Download Mirror output artefacts
   ansible.builtin.set_fact:
     infra__type: "{{ infra__type }}"
-    infra__region: "{{ infra__region }}"
+    infra__region: "{{ infra__region }}"
+
+- name: Discover VPC Dependencies during Purge cleanup
+  when:
+    - infra__force_teardown | bool
+    - infra__aws_vpc_id | length > 1
+  block:
+    - name: Fetch Network Interfaces in VPC
+      register: __infra_vpc_enis
+      amazon.aws.ec2_eni_info:
+        region: "{{ infra__region }}"
+        filters: "{{ __filters | items2dict }}"
+      vars:
+        __filters:
+          - key: "vpc-id"
+            value: "{{ infra__aws_vpc_id }}"
+
+    - name: List all EC2 instances in VPC
+      register: __infra_vpc_ec2_instances
+      community.aws.ec2_instance_info:
+        region: "{{ infra__region }}"
+        filters: "{{ __filters | items2dict }}"
+      vars:
+        __filters:
+          - key: "vpc-id"
+            value: "{{ infra__aws_vpc_id }}"
+
+    - name: Update discovered compute inventory for later action
+      ansible.builtin.set_fact:
+        infra__discovered_compute_inventory: "{{ infra__discovered_compute_inventory + __infra_vpc_ec2_instances.instances }}"
+
+    - name: List EKS Clusters
+      register: __infra_eks_cluster_list
+      command: "aws eks list-clusters"
+
+    - name: Describe all EKS Clusters
+      register: __infra_eks_cluster_details
+      ansible.builtin.command: "aws eks describe-cluster --name {{ __infra_eks_cluster_item }}"
+      loop_control:
+        loop_var: __infra_eks_cluster_item
+      loop: "{{ __infra_eks_cluster_list.stdout | from_json | community.general.json_query('clusters') }} "
+
+    - name: Filter EKS by this VPC
+      ansible.builtin.set_fact:
+        __infra_vpc_eks_cluster_names: "{{ __infra_vpc_eks_cluster_names | d([]) + [__infra_lookup_name] }}"
+      when: __infra_lookup_vpc == infra__aws_vpc_id
+      vars:
+        __infra_lookup_vpc: "{{ __infra_eks_cluster_detail_item.stdout | from_json | community.general.json_query('cluster.resourcesVpcConfig.vpcId') }}"
+        __infra_lookup_name: "{{ __infra_eks_cluster_detail_item.stdout | from_json | community.general.json_query('cluster.name') }}"
+      loop: "{{ __infra_eks_cluster_details.results }}"
+      loop_control:
+        loop_var: __infra_eks_cluster_detail_item
+
+    - name: List all ASGs in region
+      register: __infra_ec2_asg_list
+      community.aws.ec2_asg_info:
+        region: "{{ infra__region }}"
+
+    - name: Fetch list of all Subnets in VPC
+      register: __infra_disc_subnets
+      amazon.aws.ec2_vpc_subnet_info:
+        filters:
+          vpc-id: "{{ infra__aws_vpc_id }}"
+
+    - name: Filter ASGs by Subnets in this VPC
+      when:
+        - __infra_ec2_asg_list.results | length > 0
+        - __infra_asg_filter_item.vpc_zone_identifier in __infra_vpc_subnet_ids
+      ansible.builtin.set_fact:
+        __infra_aws_asg_names: "{{ __infra_aws_asg_names | default([]) + [__infra_asg_filter_item.auto_scaling_group_name] }}"
+      vars:
+        __infra_vpc_subnet_ids: "{{ __infra_disc_subnets.subnets | community.general.json_query('[*].id') | list | unique }}"
+        __infra_vpc_sg_ids: "{{ __infra_disc_sgs.security_groups | community.general.json_query('[*].group_id') | list | unique }}"
+      loop: "{{ __infra_ec2_asg_list.results }}"
+      loop_control:
+        loop_var: __infra_asg_filter_item
+
+    - name: List all AWS ELBs in region
+      community.aws.ec2_elb_info:
+        region: "{{ infra__region }}"
+      register: __infra_ec2_elbs
+
+    - name: Filter list of ELBS to match our VPC
+      when:
+        - __infra_ec2_elbs.elbs | length > 0
+        - __infra_ec2_elb_item.vpc_id == infra__aws_vpc_id
+      ansible.builtin.set_fact:
+        __infra_ec2_elb_names: "{{ __infra_ec2_elb_names | default([]) + [__infra_ec2_elb_item.name] }}"
+      loop: "{{ __infra_ec2_elbs.elbs }}"
+      loop_control:
+        loop_var: __infra_ec2_elb_item
+
+    - name: List NAT gatways in VPC
+      community.aws.ec2_vpc_nat_gateway_info:
+        region: "{{ infra__region }}"
+        filters:
+          vpc-id: "{{ infra__aws_vpc_id }}"
+      register: __infra_aws_nat_gateways
+
+    - name: List all Security Groups in VPC
+      register:  __infra_aws_sgs
+      amazon.aws.ec2_group_info:
+        region: "{{ infra__region }}"
+        filters:
+          vpc-id: "{{ infra__aws_vpc_id }}"
+
+    - name: List all Route tables in VPC
+      register: __infra_aws_rtbs
+      community.aws.ec2_vpc_route_table_info:
+        region: "{{ infra__region }}"
+        filters:
+          vpc-id: "{{ infra__aws_vpc_id }}"
+
+- name: Discover Storage during Purge cleanup
+  when:
+    - infra__force_teardown | bool
+  block:
+    - name: List AWS EBS Volumes
+      register: __infra_aws_ebs_vols
+      amazon.aws.ec2_vol_info:
+        region: "{{ infra__region }}"
+        filters:
+          status: available
+
+    - name: Check for Orphaned EFS matching Namespace
+      register: __infra_efs_fs
+      community.aws.efs_info:
+        region: "{{ infra__region }}"
+        tags:
+          Environment: "{{ infra__env_name }}"

roles/infrastructure/tasks/initialize_azure.yml

@@ -51,10 +51,10 @@
 
 - name: Set Azure Caller Information
   ansible.builtin.set_fact:
-    infra__azure_subscription_id: "{{ __azure_account_info.stdout | from_json | json_query('id') }}"
-    infra__azure_subscription_name: "{{ __azure_account_info.stdout | from_json | json_query('name') }}"
-    infra__azure_tenant_id: "{{ __azure_account_info.stdout | from_json | json_query('tenantId') }}"
-    infra__azure_calling_user: "{{ __azure_account_info.stdout | from_json | json_query('user.name') }}"
+    infra__azure_subscription_id: "{{ __azure_account_info.stdout | from_json | community.general.json_query('id') }}"
+    infra__azure_subscription_name: "{{ __azure_account_info.stdout | from_json | community.general.json_query('name') }}"
+    infra__azure_tenant_id: "{{ __azure_account_info.stdout | from_json | community.general.json_query('tenantId') }}"
+    infra__azure_calling_user: "{{ __azure_account_info.stdout | from_json | community.general.json_query('user.name') }}"
 
 - name: Print Azure Account Info
   ansible.builtin.debug:

roles/infrastructure/tasks/setup_aws_storage.yml

@@ -14,6 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Warn user about HeadBucket Forbidden errors
+  debug:
+    msg:
+      - "If the 'Create AWS Buckets' task below fails. "
+      - "If you see HeadBucket Forbidden errors. "
+      - "Your bucket name is probably use in another AWS account and you should use a different name_prefix"
+
 - name: Create AWS Buckets
   amazon.aws.aws_s3:
     region: "{{ infra__region }}"

roles/infrastructure/tasks/setup_azure_network.yml

@@ -48,8 +48,8 @@
 
 - name: Set fact for Azure Security Group IDs
   ansible.builtin.set_fact:
-    infra__azure_security_group_knox_uri: "{{ __azure_security_group_info | json_query(knox) | first }}"
-    infra__azure_security_group_default_uri: "{{ __azure_security_group_info | json_query(default) | first }}"
+    infra__azure_security_group_knox_uri: "{{ __azure_security_group_info | community.general.json_query(knox) | first }}"
+    infra__azure_security_group_default_uri: "{{ __azure_security_group_info | community.general.json_query(default) | first }}"
   vars:
     knox: "results[?__azure_network_security_group_name_item=='{{ infra__security_group_knox_name }}'].state.id"
     default: "results[?__azure_network_security_group_name_item=='{{ infra__security_group_default_name }}'].state.id"

roles/infrastructure/tasks/teardown_aws_compute.yml

@@ -14,10 +14,53 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Handle EC2 Autoscaling Groups removal if discovered
+  when:
+    - infra__force_teardown | bool
+    - __infra_aws_asg_names is defined
+    - __infra_aws_asg_names | length > 0
+  community.aws.ec2_asg:
+    name: "{{ __infra_asg_remove_item }}"
+    state: absent
+  loop: "{{ __infra_aws_asg_names }}"
+  loop_control:
+    loop_var: __infra_asg_remove_item
+
+- name: Handle EKS Cluster removal if discovered
+  when:
+    - infra__force_teardown | bool
+    - __infra_vpc_eks_cluster_names is defined
+    - __infra_vpc_eks_cluster_names | length > 0
+  community.aws.aws_eks_cluster:
+    name: "{{ __infra_eks_remove_item }}"
+    wait: yes
+    state: absent
+  loop: "{{ __infra_vpc_eks_cluster_names }}"
+  loop_control:
+    loop_var: __infra_eks_remove_item
+  async: 3600 # 1 hour timeout
+  poll: 0
+  register: __eks_teardowns_info
+
+- name: Wait for EKS teardowns to complete
+  when:
+    - __eks_teardowns_info is defined
+    - __eks_teardowns_info.results is defined
+    - __eks_teardowns_info.results | length > 0
+  ansible.builtin.async_status:
+    jid: "{{ __eks_teardown.ansible_job_id }}"
+  loop_control:
+    loop_var: __eks_teardown
+  loop: "{{ __eks_teardowns_info.results }}"
+  register: __eks_teardowns_async
+  until: __eks_teardowns_async.finished
+  retries: 120
+  delay: 10
+
 - name: Handle Compute Removal if Discovered
   when: infra__discovered_compute_inventory | count > 0
   amazon.aws.ec2:
     region: "{{ infra__region }}"
     wait: yes
     state: absent
-    instance_ids: "{{ infra__discovered_compute_inventory | community.general.json_query('[*].instance_id') | list }}"
+    instance_ids: "{{ infra__discovered_compute_inventory | community.general.json_query('[*].instance_id') | list | unique }}"