Add Terraform deployment engine for cloud resources (#65)

* Add parameters for Terraform deployment engine
* Update README with details of Terraform deployment
* Use ansible tags to determine action on remote state resources
* Consolidate Teardown tasks in main.yml into a single play

Signed-off-by: Jim Enright <jenright@cloudera.com>

Author: jimright

main.yml

@@ -28,7 +28,7 @@
 - name: Import the Cloud Interactions Playbook
   import_playbook: cloud.yml
 
-- name: Teardown Dynamic Inventory when requested
+- name: Teardown Cleanup
   hosts: localhost
   tags: [teardown,never]
   gather_facts: yes
@@ -38,6 +38,15 @@
         name: cloudera_deploy
         tasks_from: clean_dynamic_inventory
 
+    - name: Remove Terraform remote state resources if requested
+      when:
+        - globals.infra_deployment_engine == 'terraform'
+        - globals.terraform_auto_remote_state | bool
+        - globals.terraform_state_storage in ['remote_s3']
+      ansible.builtin.include_role:
+        name: cloudera_deploy
+        tasks_from: auto_terraform_state
+
 - name: Prepare for Cloudera Cluster Run
   hosts: localhost
   tags: always

readme.adoc

@@ -447,6 +447,46 @@ WARNING: Setting a deployment to a lower runlevel, e.g. from `run` to `infra` wi
 
 For further details on the various _runlevel_-like tags for CDP Public Cloud, see the https://github.com/cloudera-labs/cloudera.exe/blob/main/docs/runlevels.md[Runlevel Guide] in the `cloudera.exe` project.
 
+=== Terraform Deployment Engine
+
+Terraform can optionally be used to create the cloud infrastructure. This will attempt to create the cloud provider assets at the `infra` (network, storage and compute) and `plat` (IAM policies and roles) runlevels using Terraform resources. A list of Terraform related parameters are shown in the table below. 
+
+.List of parameters used by Terraform deployment engine
+[cols="1,1,1,1"]
+|===
+|Parameter|Description|Default Value|Notes
+
+|`infra_deployment_engine`
+|The engine (ansible or terraform) that will be used to create the infrastructure resources.
+| `ansible`
+| Needs to be set to `terraform` for Terraform-deployment.
+
+|`terraform_base_dir`
+| Top-level directory where all Terraform assets will be placed. Includes processed Jinja template files for Terraform,  timestamped artefact of Terraform files and the workspace directory where terraform apply/destroy is run.
+| `~/.config/cloudera-deploy/terraform`
+|
+
+|`terraform_state_storage`
+|The type of backend storage to use for the Terraform state. 
+| `local`
+| Current options are `local` or `remote_s3`
+
+|`terraform_auto_remote_state`
+| Flag to allow Cloudera Deploy automatically provision remote state resources as part of its initialization. This will also teardown these resources during cleanup.
+| `False`
+|
+
+|`terraform_remote_state_bucket`
+|The name of the Terraform state storage bucket. 
+| 
+| Required if using `remote_s3` state storage. Value is derived from `name_prefix` if terraform_auto_remote_state is True.
+
+|`terraform_remote_state_lock_table`
+|The name of the table to track locks of remote Terraform state.
+| 
+| Required if using `remote_s3` state storage. Value is derived from `name_prefix` if terraform_auto_remote_state is True.
+|===
+
 == Definitions
 
 Cloudera Deploy uses a set of configuration files within a directory to define and coordinate a deployment. This directory also stores any artifacts created during the deployment, such as Ansible inventory files, CDP environment readouts, etc.

roles/cloudera_deploy/defaults/main.yml

@@ -47,5 +47,9 @@ default_parcel_distro: el7  # el8, bionic, focal
 default_download_link_expiry: 3600
 
 # Default Deployment Controls
+default_infra_deployment_engine: ansible
 default_infra_type: aws
 default_infra_region: us-east-1
+
+# Terraform defaults
+default_terraform_base_dir: "{{ [default_config_path, 'terraform'] | path_join }}" 

roles/cloudera_deploy/tasks/auto_terraform_state.yml

@@ -0,0 +1,48 @@
+---
+
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Resources for remote_s3 state storage
+  when:
+    - globals.terraform_state_storage == 'remote_s3'
+  block:
+    
+    # Create or Teardown the resources
+    - name: AWS Bucket for Remote State Storage
+      amazon.aws.aws_s3:
+        region: "{{ globals.region }}"
+        bucket: "{{ globals.terraform_remote_state_bucket}}"
+        mode: "{{ ('teardown' not in ansible_run_tags)  | ternary('create', 'delete') }}" # Check ansible tag to determine action
+        permission: private
+      register: __infra_aws_storage_locations_info
+
+    - name: AWS DynamoDB for Remote State Locking
+      community.aws.dynamodb_table:
+        region: "{{ globals.region }}"
+        name: "{{ globals.terraform_remote_state_lock_table }}"
+        read_capacity: 1
+        write_capacity: 1
+        hash_key_name: LockID
+        hash_key_type: STRING
+        state: "{{ ('teardown' not in ansible_run_tags) | ternary('present', 'absent') }}" # Check ansible tag to determine action
+
+    - name: Print remote state configuration
+      when: "'teardown' not in ansible_run_tags"
+      ansible.builtin.debug:
+        msg:
+          - "Resources for remote_s3 Terraform State created."
+          - "S3 Bucket Name: {{ globals.terraform_remote_state_bucket}}"
+          - "DynamoDB Locking Table: {{ globals.terraform_remote_state_lock_table}}"
+        verbosity: 3

roles/cloudera_deploy/tasks/init.yml

@@ -173,7 +173,13 @@
       name_prefix: "{{ name_prefix | default(default_name_prefix) }}"
       tags: "{{ tags | default(omit) }}"
       region: "{{ infra_region | default(default_infra_region) }}"
+      infra_deployment_engine: "{{ infra_deployment_engine | default(default_infra_deployment_engine) }}"
       infra_type: "{{ infra_type | default(default_infra_type) }}"
+      terraform_base_dir: "{{ terraform_base_dir | default(default_terraform_base_dir) | expanduser }}"
+      terraform_state_storage: "{{ terraform_state_storage | default(omit) }}"
+      terraform_auto_remote_state: "{{ terraform_auto_remote_state | default(False) }}"
+      terraform_remote_state_bucket: "{{ terraform_remote_state_bucket | default(omit) }}"
+      terraform_remote_state_lock_table: "{{ terraform_remote_state_lock_table | default(omit) }}"
       ssh:
         public_key_id: "{{ public_key_id | default(omit) }}"
         public_key_file: "{{ public_key_file | default(omit) }}"
@@ -217,6 +223,31 @@
     fail_msg: "You must supply a valid Namespace"
     quiet: yes
 
+- name: Check Deployment Engine variable
+  ansible.builtin.assert:
+    that:
+      - globals.infra_deployment_engine in ['ansible', 'terraform']
+    fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'"
+    
+- name: Check Supplied terraform_base_dir variable
+  when:
+    - infra_deployment_engine == 'terraform'
+  ansible.builtin.assert:
+    that:
+      - globals.terraform_base_dir is defined
+      - globals.terraform_base_dir | length > 0
+    fail_msg: "You must supply a 'terraform_base_dir' where Terraform assets will be placed"
+    quiet: yes
+
+- name: Check Supplied terraform_auto_remote_state variable
+  when:
+    - infra_deployment_engine == 'terraform'
+  ansible.builtin.assert:
+    that:
+      - (globals.terraform_auto_remote_state|bool is sameas true) or (globals.terraform_auto_remote_state|bool is sameas false)
+    fail_msg: "The terraform_auto_remote_state variable must be a boolean variable"
+    quiet: yes
+
 # SSH
 - name: Use default SSH public key id
   when: globals.ssh.public_key_id is undefined
@@ -405,3 +436,44 @@
     fail_msg: >-
       Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars.
     quiet: yes
+
+# Provision Terraform remote state resources if requested
+- name: Initialize Terraform remote state resources
+  when:
+    - init__call_cloud_role | bool
+    - globals.infra_deployment_engine == 'terraform'
+    - globals.terraform_auto_remote_state | bool
+    - globals.terraform_state_storage in ['remote_s3']
+  block:
+    # Set resource variable names if not already done
+    - name: Set variables for remote state bucket if not set
+      when: (globals.terraform_remote_state_bucket is not defined) or
+            ( (globals.terraform_remote_state_bucket) | length == 0)
+      ansible.builtin.set_fact:
+        globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}"
+      vars:
+        remote_state_vars:
+          terraform_remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}"
+
+    - name: Set variables for remote state lock table if not set
+      when: (globals.terraform_remote_state_lock_table is not defined) or
+            (globals.terraform_remote_state_lock_table | length == 0)   
+      ansible.builtin.set_fact:
+        globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}"
+      vars:
+        remote_state_vars:
+          terraform_remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}"
+
+    - name: Create remote state resources
+      when: "'teardown' not in ansible_run_tags"
+      ansible.builtin.include_role:
+        name: cloudera_deploy
+        tasks_from: auto_terraform_state
+
+- name: Prepare for Download Mirror Population if requested and necessary
+  when:
+    - use_download_mirror | default(default_enable_download_mirror) | bool
+    - "'teardown' not in ansible_run_tags"
+  ansible.builtin.include_role:
+    name: cloudera_deploy
+    tasks_from: prepare_download_mirror