Skip to content

Commit d8895b6

Browse files
ChaffelsonChaffelson
authored
Allow skipping of GCP Service and IAM management (#130)
* Add options to control whether Ansible will manage GCP identities, roles, services, and policies for the user. These commands fail if the user does not have permission to change that setting, whether already enabled/created or not Added env.gcp.manage_identities to control deploy and teardown of GCP credentials, identities and roles. Defaults to True to match existing behavior Added env.gcp.auto_enable_services to control enabling of required GCP services for CDP. Defaults to True to match existing behavior * Update setup_gcp_authz.yml * Removed leftover pause from debugging Signed-off-by: Daniel Chaffelson <chaffelson@gmail.com>
1 parent 5a1f9eb commit d8895b6

File tree

5 files changed

+277
-263
lines changed

5 files changed

+277
-263
lines changed

‎docs/configuration.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,8 @@ env:
197197
version:
198198
scale:
199199
gcp:
200+
auto_enable_services:
201+
manage_identities:
200202
bindings:
201203
cross_account:
202204
logs:

‎roles/platform/defaults/main.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,9 @@ plat__gcp_storage_location_logs: "{{ env.gcp.storage.path.logs | de
169169
plat__gcp_xaccount_policy_bindings: "{{ env.gcp.bindings.cross_account | default(plat__gcp_xaccount_policy_bindings_default) }}"
170170
plat__gcp_log_role_perms: "{{ env.gcp.bindings.logs | default(plat__gcp_log_policy_bindings_default) }}"
171171

172+
plat__gcp_manage_identities: "{{ env.gcp.manage_identities | default(true) }}"
173+
plat__gcp_enable_services: "{{ env.gcp.auto_enable_services | default(true) }}"
174+
172175
# Azure
173176
plat__azure_app_suffix: "{{ env.azure.app.suffix | default(common__app_suffix) }}"
174177
plat__azure_role_suffix: "{{ env.azure.role.suffix | default(common__role_suffix) }}"

‎roles/platform/tasks/initialize_setup_gcp.yml‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,25 @@
1515
# limitations under the License.
1616

1717
# https://docs.cloudera.com/management-console/cloud/requirements-gcp/topics/mc-gcp_apis.html
18-
- name: Fetch listing of Enabled GCP Service APIs
19-
command: >
20-
gcloud services list --enabled --project {{ plat__gcp_project }}
21-
register: __gcp_services_info
18+
- name: Ensure Google Services Enabled
19+
when: plat__gcp_enable_services | bool
20+
block:
21+
- name: Fetch list of enabled GCP Services
22+
command: >
23+
gcloud services list --enabled --project {{ plat__gcp_project }}
24+
register: __gcp_services_info
2225

23-
- name: Determine list of missing APIs
24-
set_fact:
25-
__plat_gcp_services_to_enable: "{{ __plat_gcp_services_to_enable | default([]) + ([__gcp_service_item] if __gcp_service_item not in __gcp_services_info.stdout else []) | unique }}"
26-
loop: "{{ plat__gcp_required_services }}"
27-
loop_control:
28-
loop_var: __gcp_service_item
26+
- name: Determine list of missing Services
27+
set_fact:
28+
__plat_gcp_services_to_enable: "{{ __plat_gcp_services_to_enable | default([]) + ([__gcp_service_item] if __gcp_service_item not in __gcp_services_info.stdout else []) | unique }}"
29+
loop: "{{ plat__gcp_required_services }}"
30+
loop_control:
31+
loop_var: __gcp_service_item
2932

30-
- name: Enable missing GCP Service APIs
31-
when: __plat_gcp_services_to_enable | length > 0
32-
command: >
33-
gcloud services enable --quiet {{ __gcp_enable_item }}
34-
loop: "{{ __plat_gcp_services_to_enable }}"
35-
loop_control:
36-
loop_var: __gcp_enable_item
33+
- name: Enable missing GCP Service APIs
34+
when: __plat_gcp_services_to_enable | length > 0
35+
command: >
36+
gcloud services enable --quiet {{ __gcp_enable_item }}
37+
loop: "{{ __plat_gcp_services_to_enable }}"
38+
loop_control:
39+
loop_var: __gcp_enable_item

‎roles/platform/tasks/setup_gcp_authz.yml‎

Lines changed: 150 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -14,161 +14,164 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
- name: Create a temporary directory for Credentials
18-
ansible.builtin.tempfile:
19-
prefix: "gcp-creds-"
20-
state: directory
21-
register: __gcp_creds_tmpdir
17+
- name: Create or update GCP Identities and Roles
18+
when: plat__gcp_manage_identities | bool
19+
block:
20+
- name: Create a temporary directory for Credentials
21+
ansible.builtin.tempfile:
22+
prefix: "gcp-creds-"
23+
state: directory
24+
register: __gcp_creds_tmpdir
2225

23-
- name: Create GCP xaccount service account
24-
register: __gcp_xaccount_sa_info
25-
google.cloud.gcp_iam_service_account:
26-
name: "{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
27-
display_name: "{{ plat__gcp_xaccount_identity_name }}"
28-
project: "{{ plat__gcp_project }}"
29-
state: present
26+
- name: Create GCP xaccount service account
27+
register: __gcp_xaccount_sa_info
28+
google.cloud.gcp_iam_service_account:
29+
name: "{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
30+
display_name: "{{ plat__gcp_xaccount_identity_name }}"
31+
project: "{{ plat__gcp_project }}"
32+
state: present
3033

31-
- name: Set GCP Cross Account Bindings
32-
register: __gcp_xaccount_bind_info
33-
loop_control:
34-
loop_var: __gcp_sa_binding_item
35-
loop: "{{ plat__gcp_xaccount_policy_bindings }}"
36-
command: >
37-
gcloud projects
38-
add-iam-policy-binding {{ plat__gcp_project }}
39-
--member="serviceAccount:{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
40-
--role={{ __gcp_sa_binding_item |quote }}
41-
--no-user-output-enabled
42-
--condition=None
34+
- name: Set GCP Cross Account Bindings
35+
register: __gcp_xaccount_bind_info
36+
loop_control:
37+
loop_var: __gcp_sa_binding_item
38+
loop: "{{ plat__gcp_xaccount_policy_bindings }}"
39+
command: >
40+
gcloud projects
41+
add-iam-policy-binding {{ plat__gcp_project }}
42+
--member="serviceAccount:{{ plat__gcp_xaccount_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
43+
--role={{ __gcp_sa_binding_item |quote }}
44+
--no-user-output-enabled
45+
--condition=None
4346
44-
- name: Generate Key for Google Cross Account Service Account
45-
google.cloud.gcp_iam_service_account_key:
46-
service_account: "{{ __gcp_xaccount_sa_info }}"
47-
private_key_type: TYPE_GOOGLE_CREDENTIALS_FILE
48-
path: "{{ __gcp_creds_tmpdir.path }}/{{ plat__gcp_xaccount_identity_name }}-gcp-cred.json"
49-
project: "{{ plat__gcp_project }}"
50-
state: present
47+
- name: Generate Key for Google Cross Account Service Account
48+
google.cloud.gcp_iam_service_account_key:
49+
service_account: "{{ __gcp_xaccount_sa_info }}"
50+
private_key_type: TYPE_GOOGLE_CREDENTIALS_FILE
51+
path: "{{ __gcp_creds_tmpdir.path }}/{{ plat__gcp_xaccount_identity_name }}-gcp-cred.json"
52+
project: "{{ plat__gcp_project }}"
53+
state: present
5154

52-
- name: Create Cross Account Google CDP Credential
53-
cloudera.cloud.env_cred:
54-
state: present
55-
cloud: "{{ plat__infra_type }}"
56-
name: "{{ plat__xacccount_credential_name }}"
57-
secret: "{{ __gcp_creds_tmpdir.path }}/{{ plat__gcp_xaccount_identity_name }}-gcp-cred.json"
55+
- name: Create Cross Account Google CDP Credential
56+
cloudera.cloud.env_cred:
57+
state: present
58+
cloud: "{{ plat__infra_type }}"
59+
name: "{{ plat__xacccount_credential_name }}"
60+
secret: "{{ __gcp_creds_tmpdir.path }}/{{ plat__gcp_xaccount_identity_name }}-gcp-cred.json"
5861

59-
- name: Create Custom GCP Log Role
60-
register: __gcp_role_creation_info
61-
google.cloud.gcp_iam_role:
62-
name: "{{ plat__gcp_log_role_name }}"
63-
title: "{{ plat__gcp_log_role_name }}"
64-
description: "{{ plat__gcp_log_role_name }}"
65-
included_permissions: "{{ plat__gcp_log_role_perms }}"
66-
project: "{{ plat__gcp_project }}"
67-
state: present
68-
failed_when:
69-
- __gcp_role_creation_info.msg is defined
70-
- "'GCP returned error' in __gcp_role_creation_info.msg"
71-
- "'marked for deletion' not in __gcp_role_creation_info.msg"
62+
- name: Create Custom GCP Log Role
63+
register: __gcp_role_creation_info
64+
google.cloud.gcp_iam_role:
65+
name: "{{ plat__gcp_log_role_name }}"
66+
title: "{{ plat__gcp_log_role_name }}"
67+
description: "{{ plat__gcp_log_role_name }}"
68+
included_permissions: "{{ plat__gcp_log_role_perms }}"
69+
project: "{{ plat__gcp_project }}"
70+
state: present
71+
failed_when:
72+
- __gcp_role_creation_info.msg is defined
73+
- "'GCP returned error' in __gcp_role_creation_info.msg"
74+
- "'marked for deletion' not in __gcp_role_creation_info.msg"
7275

73-
- name: Undelete Custom Log role if recently marked for deletion
74-
when:
75-
- __gcp_role_creation_info.deleted is defined
76-
- __gcp_role_creation_info.deleted | bool
77-
ansible.builtin.command: >
78-
gcloud iam roles
79-
undelete {{ plat__gcp_log_role_name }}
80-
--project={{ plat__gcp_project }}
81-
register: __gcp_role_undelete
76+
- name: Undelete Custom Log role if recently marked for deletion
77+
when:
78+
- __gcp_role_creation_info.deleted is defined
79+
- __gcp_role_creation_info.deleted | bool
80+
ansible.builtin.command: >
81+
gcloud iam roles
82+
undelete {{ plat__gcp_log_role_name }}
83+
--project={{ plat__gcp_project }}
84+
register: __gcp_role_undelete
8285

83-
- name: Fetch resulting Custom GCP Log Role status
84-
ansible.builtin.command: >
85-
gcloud iam roles
86-
describe {{ plat__gcp_log_role_name }}
87-
--project={{ plat__gcp_project }}
88-
register: __gcp_custom_log_role_info
89-
failed_when:
90-
- __gcp_custom_log_role_info.rc == 1
91-
- "'NOT_FOUND:' not in __gcp_custom_log_role_info.stderr"
86+
- name: Fetch resulting Custom GCP Log Role status
87+
ansible.builtin.command: >
88+
gcloud iam roles
89+
describe {{ plat__gcp_log_role_name }}
90+
--project={{ plat__gcp_project }}
91+
register: __gcp_custom_log_role_info
92+
failed_when:
93+
- __gcp_custom_log_role_info.rc == 1
94+
- "'NOT_FOUND:' not in __gcp_custom_log_role_info.stderr"
9295

93-
- name: Fail when custom GCP Log Role is not available
94-
ansible.builtin.assert:
95-
that:
96-
- "'includedPermissions:' in __gcp_custom_log_role_info.stdout"
97-
- "'NOT_FOUND' not in __gcp_custom_log_role_info.stderr"
98-
quiet: yes
99-
fail_msg: |
100-
Custom Log Role {{ plat__gcp_log_role_name }} could not be created or undeleted.
101-
It is likely that the unique role_id was marked for deletion recently and you are in the GCloud no-reuse window.
102-
This no-reuse of a role_id window is typically somewhere from 7-14 days after initial deletion, and lasts 30 days.
103-
Please use a different name for your custom Role.
96+
- name: Fail when custom GCP Log Role is not available
97+
ansible.builtin.assert:
98+
that:
99+
- "'includedPermissions:' in __gcp_custom_log_role_info.stdout"
100+
- "'NOT_FOUND' not in __gcp_custom_log_role_info.stderr"
101+
quiet: yes
102+
fail_msg: |
103+
Custom Log Role {{ plat__gcp_log_role_name }} could not be created or undeleted.
104+
It is likely that the unique role_id was marked for deletion recently and you are in the GCloud no-reuse window.
105+
This no-reuse of a role_id window is typically somewhere from 7-14 days after initial deletion, and lasts 30 days.
106+
Please use a different name for your custom Role.
104107
105-
- name: Create Operational GCP Service Accounts
106-
register: __gcp_identity_info
107-
loop_control:
108-
loop_var: __gcp_identity_item
109-
label: __gcp_identity_item
110-
loop:
111-
- "{{ plat__gcp_log_identity_name }}"
112-
- "{{ plat__gcp_datalakeadmin_identity_name }}"
113-
- "{{ plat__gcp_ranger_audit_identity_name }}"
114-
- "{{ plat__gcp_idbroker_identity_name }}"
115-
google.cloud.gcp_iam_service_account:
116-
name: "{{ __gcp_identity_item }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
117-
display_name: "{{ __gcp_identity_item }}"
118-
project: "{{ plat__gcp_project }}"
119-
state: present
108+
- name: Create Operational GCP Service Accounts
109+
register: __gcp_identity_info
110+
loop_control:
111+
loop_var: __gcp_identity_item
112+
label: __gcp_identity_item
113+
loop:
114+
- "{{ plat__gcp_log_identity_name }}"
115+
- "{{ plat__gcp_datalakeadmin_identity_name }}"
116+
- "{{ plat__gcp_ranger_audit_identity_name }}"
117+
- "{{ plat__gcp_idbroker_identity_name }}"
118+
google.cloud.gcp_iam_service_account:
119+
name: "{{ __gcp_identity_item }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
120+
display_name: "{{ __gcp_identity_item }}"
121+
project: "{{ plat__gcp_project }}"
122+
state: present
120123

121-
- name: Set Operational Service Account Policy Bindings
122-
loop_control:
123-
loop_var: __gcp_binding_item
124-
label: __gcp_binding_item.member
125-
loop:
126-
# Logs
127-
- member: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
128-
role: "projects/{{ plat__gcp_project }}/roles/{{ plat__gcp_log_role_name }}"
129-
condition: "expression=resource.name == '{{ plat__gcp_log_role_name }}',title='{{ plat__gcp_log_role_name }}'"
130-
# Data Access
131-
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
132-
role: "{{ plat__gcp_roles.storage_admin }}"
133-
condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}'"
134-
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
135-
role: "{{ plat__gcp_roles.storage_admin }}"
136-
condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}-fulladmin'"
137-
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
138-
role: "{{ plat__gcp_roles.storage_object_admin }}"
139-
condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}'"
140-
# Ranger Audit
141-
- member: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
142-
role: "{{ plat__gcp_roles.storage_object_admin }}"
143-
condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}-fulladmin'"
144-
# ID Broker / Assumer Role
145-
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
146-
role: "{{ plat__gcp_roles.iam_workload_identity_user }}"
147-
condition: "expression=resource.name.startsWith('{{ plat__namespace }}'),title='{{ plat__gcp_idbroker_identity_name }}'"
148-
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
149-
role: "{{ plat__gcp_roles.iam_service_account_user }}"
150-
condition: "None"
151-
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
152-
role: "{{ plat__gcp_roles.iam_service_account_token_creator }}"
153-
condition: "None"
154-
command: >
155-
gcloud projects
156-
add-iam-policy-binding {{ plat__gcp_project }}
157-
--member={{ __gcp_binding_item.member |quote }}
158-
--role={{ __gcp_binding_item.role |quote }}
159-
--condition={{ __gcp_binding_item.condition |quote }}
124+
- name: Set Operational Service Account Policy Bindings
125+
loop_control:
126+
loop_var: __gcp_binding_item
127+
label: __gcp_binding_item.member
128+
loop:
129+
# Logs
130+
- member: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
131+
role: "projects/{{ plat__gcp_project }}/roles/{{ plat__gcp_log_role_name }}"
132+
condition: "expression=resource.name == '{{ plat__gcp_log_role_name }}',title='{{ plat__gcp_log_role_name }}'"
133+
# Data Access
134+
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
135+
role: "{{ plat__gcp_roles.storage_admin }}"
136+
condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}'"
137+
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
138+
role: "{{ plat__gcp_roles.storage_admin }}"
139+
condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_datalakeadmin_identity_name }}-fulladmin'"
140+
- member: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
141+
role: "{{ plat__gcp_roles.storage_object_admin }}"
142+
condition: "expression=resource.name == '{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}'"
143+
# Ranger Audit
144+
- member: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
145+
role: "{{ plat__gcp_roles.storage_object_admin }}"
146+
condition: "expression=resource.name == '//storage.googleapis.com/projects/_/buckets/{{ plat__gcp_storage_location_data }}',title='{{ plat__gcp_ranger_audit_identity_name }}-fulladmin'"
147+
# ID Broker / Assumer Role
148+
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
149+
role: "{{ plat__gcp_roles.iam_workload_identity_user }}"
150+
condition: "expression=resource.name.startsWith('{{ plat__namespace }}'),title='{{ plat__gcp_idbroker_identity_name }}'"
151+
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
152+
role: "{{ plat__gcp_roles.iam_service_account_user }}"
153+
condition: "None"
154+
- member: "serviceAccount:{{ plat__gcp_idbroker_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com"
155+
role: "{{ plat__gcp_roles.iam_service_account_token_creator }}"
156+
condition: "None"
157+
command: >
158+
gcloud projects
159+
add-iam-policy-binding {{ plat__gcp_project }}
160+
--member={{ __gcp_binding_item.member |quote }}
161+
--role={{ __gcp_binding_item.role |quote }}
162+
--condition={{ __gcp_binding_item.condition |quote }}
160163
161-
- name: Add Service Accounts to Storage Policies for Buckets
162-
loop_control:
163-
loop_var: __gcp_pol_item
164-
loop:
165-
- account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
166-
bucket: "{{ plat__gcp_storage_location_logs }}"
167-
- account: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
168-
bucket: "{{ plat__gcp_storage_location_data }}"
169-
- account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
170-
bucket: "{{ plat__gcp_storage_location_data }}"
171-
command: >
172-
gsutil iam
173-
ch {{ __gcp_pol_item.account |quote }}
174-
gs://{{ __gcp_pol_item.bucket |quote }}
164+
- name: Add Service Accounts to Storage Policies for Buckets
165+
loop_control:
166+
loop_var: __gcp_pol_item
167+
loop:
168+
- account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
169+
bucket: "{{ plat__gcp_storage_location_logs }}"
170+
- account: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
171+
bucket: "{{ plat__gcp_storage_location_data }}"
172+
- account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin"
173+
bucket: "{{ plat__gcp_storage_location_data }}"
174+
command: >
175+
gsutil iam
176+
ch {{ __gcp_pol_item.account |quote }}
177+
gs://{{ __gcp_pol_item.bucket |quote }}

0 commit comments

Comments
 (0)