Improve DataFlow Integration (#20)

* Rework DF Disable function and force deletion for better determinism.
* Modify `df_info` to accept either a string name of DF Services, or the CRN of an individual service. This allows for situations where DF Services may be disabled but still listed when needing clean up.
* Improve DF listing mechanism in env_info to only retrieve active DF Services which need cleanup before an Environment is removed.

Signed-off-by: Daniel Chaffelson <chaffelson@gmail.com>

Author: Chaffelson

plugins/modules/df.py

@@ -269,16 +269,7 @@ def process(self):
                 if self.module.check_mode:
                     self.service = self.target
                 else:
-                    if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES:
-                        self.service = self.cdpy.df.disable_environment(
-                            env_crn=self.env_crn,
-                            persist=self.persist,
-                            force=self.force
-                        )
-                    if self.wait:
-                        self.service = self._wait_for_disabled()
-                    else:
-                        self.service = self.cdpy.df.describe_environment(env_crn=self.name)
+                    self._disable_df()
             elif self.state in ['present']:
                 self.module.warn(
                     "Dataflow Service already enabled and configuration validation and reconciliation is not supported;" +
@@ -319,11 +310,40 @@ def _wait_for_enabled(self):
             delay=self.delay, timeout=self.timeout
         )
 
-    def _wait_for_disabled(self):
-        return self.cdpy.sdk.wait_for_state(
-            describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn), field=None,
-            delay=self.delay, timeout=self.timeout
-        )
+    def _disable_df(self):
+        # Attempt clean Disable, which also ensures we have tried at least once before we do a forced removal
+        if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES:
+            self.service = self.cdpy.df.disable_environment(
+                env_crn=self.env_crn,
+                persist=self.persist
+            )
+        if self.wait:
+            # Wait for Clean Disable, if possible
+            self.service = self.cdpy.sdk.wait_for_state(
+                describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn),
+                field=['status', 'state'],
+                state=self.cdpy.sdk.STOPPED_STATES + self.cdpy.sdk.REMOVABLE_STATES + [None],
+                delay=self.delay, timeout=self.timeout, ignore_failures=True
+            )
+        else:
+            self.service = self.cdpy.df.describe_environment(env_crn=self.name)
+        # Check disable result against need for further forced delete action, in case it didn't work first time around
+        if self.service is not None:
+            if self.service['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES:
+                if self.force:
+                    self.service = self.cdpy.df.delete_environment(
+                        env_crn=self.env_crn
+                    )
+                else:
+                    self.module.fail_json(msg="DF Service Disable failed and Force delete not requested")
+            if self.wait:
+                self.service = self.cdpy.sdk.wait_for_state(
+                    describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn),
+                    field=None,  # This time we require removal or declare failure
+                    delay=self.delay, timeout=self.timeout
+                )
+            else:
+                self.service = self.cdpy.df.describe_environment(env_crn=self.name)
 
 
 def main():

plugins/modules/df_info.py

@@ -37,6 +37,7 @@
   name:
     description:
       - If a name is provided, that DataFlow Service will be described.
+      - Must be CDP Environment CRN or string name of DataFlow Service
     type: str
     required: False
     aliases:
@@ -168,9 +169,12 @@ def __init__(self, module):
     @CdpModule._Decorators.process_debug
     def process(self):
         if self.name:  # Note that both None and '' will trigger this
-            service_single = self.cdpy.df.describe_environment(name=self.name)
-            if service_single is not None:
-                self.services.append(service_single)
+            if self.name.startswith('crn:'):
+                service_single = self.cdpy.df.describe_environment(env_crn=self.name)
+                if service_single is not None:
+                    self.services.append(service_single)
+            else:
+                self.services = self.cdpy.df.list_environments(name=self.name)
         else:
             self.services = self.cdpy.df.list_environments()
 

plugins/modules/env_info.py

@@ -434,12 +434,13 @@ def process(self):
         if self.descendants and self.environments:
             updated_envs = []
             for this_env in self.environments:
+                df = self.cdpy.df.describe_environment(this_env['crn'])
                 this_env['descendants'] = {
                     'datahub': self.cdpy.datahub.describe_all_clusters(this_env['environmentName']),
                     'dw': self.cdpy.dw.gather_clusters(this_env['crn']),
                     'ml': self.cdpy.ml.describe_all_workspaces(this_env['environmentName']),
                     'opdb': self.cdpy.opdb.describe_all_databases(this_env['environmentName']),
-                    'df': [self.cdpy.df.describe_environment(this_env['crn'])]
+                    'df': df if df is not None else []
                 }
                 updated_envs.append(this_env)
             self.environments = updated_envs