Merge pull request #71 from putmanoj/retire-tf-runner-stack

Retire terraform runner stack

app/models/manageiq/providers/embedded_terraform/automation_manager/job.rb

@@ -1,11 +1,13 @@
 class ManageIQ::Providers::EmbeddedTerraform::AutomationManager::Job < Job
-  def self.create_job(template, env_vars, input_vars, credentials, poll_interval: 1.minute)
+  def self.create_job(template, env_vars, input_vars, credentials, action: ResourceAction::PROVISION, terraform_stack_id: nil, poll_interval: 1.minute)
     super(
-      :template_id   => template.id,
-      :env_vars      => env_vars,
-      :input_vars    => input_vars,
-      :credentials   => credentials,
-      :poll_interval => poll_interval,
+      :template_id        => template.id,
+      :env_vars           => env_vars,
+      :input_vars         => input_vars,
+      :credentials        => credentials,
+      :poll_interval      => poll_interval,
+      :action             => action,
+      :terraform_stack_id => terraform_stack_id
     )
   end
 
@@ -21,12 +23,28 @@ def pre_execute
   def execute
     template_path = File.join(options[:git_checkout_tempdir], template_relative_path)
     credentials   = Authentication.where(:id => options[:credentials])
-    extra_vars    = options.dig(:input_vars, :extra_vars) || {}
-
-    response = Terraform::Runner.run(decrypt_extra_vars(extra_vars), template_path, :credentials => credentials, :env_vars => options[:env_vars])
-
-    options[:terraform_stack_id] = response.stack_id
-    save!
+    input_vars    = options.dig(:input_vars, :extra_vars) || {}
+    action        = options[:action]
+
+    case action
+    when ResourceAction::RETIREMENT
+      Terraform::Runner.delete_stack(
+        options[:terraform_stack_id],
+        template_path,
+        :input_vars  => decrypt_vars(input_vars),
+        :credentials => credentials,
+        :env_vars    => options[:env_vars]
+      )
+    else
+      response = Terraform::Runner.create_stack(
+        template_path,
+        :input_vars  => decrypt_vars(input_vars),
+        :credentials => credentials,
+        :env_vars    => options[:env_vars]
+      )
+      options[:terraform_stack_id] = response.stack_id
+      save!
+    end
 
     queue_poll_runner
   end
@@ -106,9 +124,8 @@ def stack_response
     @stack_response ||= Terraform::Runner::ResponseAsync.new(options[:terraform_stack_id])
   end
 
-  def decrypt_extra_vars(extra_vars)
-    result = extra_vars.deep_dup
-    result.transform_values! { |val| val.kind_of?(String) ? ManageIQ::Password.try_decrypt(val) : val }
+  def decrypt_vars(input_vars)
+    input_vars.transform_values { |val| val.kind_of?(String) ? ManageIQ::Password.try_decrypt(val) : val }
   end
 
   def configuration_script_source

app/models/manageiq/providers/embedded_terraform/automation_manager/template.rb

@@ -4,7 +4,11 @@ class ManageIQ::Providers::EmbeddedTerraform::AutomationManager::Template < Mana
   def run(vars = {}, _userid = nil)
     env_vars    = vars.delete(:env) || {}
     credentials = vars.delete(:credentials)
+    action = vars.delete(:action) || ResourceAction::PROVISION
+    terraform_stack_id = vars.delete(:terraform_stack_id)
 
-    self.class.module_parent::Job.create_job(self, env_vars, vars, credentials).tap(&:signal_start)
+    self.class.module_parent::Job.create_job(
+      self, env_vars, vars, credentials, :action => action, :terraform_stack_id => terraform_stack_id
+    ).tap(&:signal_start)
   end
 end

app/models/service_template_terraform_template.rb

@@ -8,7 +8,7 @@ def self.default_reconfiguration_entry_point
   end
 
   def self.default_retirement_entry_point
-    '/Service/Generic/StateMachines/GenericLifecycle/Retire_Basic_Resource'
+    '/Service/Generic/StateMachines/GenericLifecycle/Retire_Advanced_Resource_None'
   end
 
   def self.create_catalog_item(options, _auth_user)
@@ -39,18 +39,26 @@ def update_catalog_item(options, auth_user = nil)
     super
   end
 
-  def self.validate_config_info(info)
+  private_class_method def self.validate_config_info(info)
     info[:provision][:fqname]   ||= default_provisioning_entry_point(SERVICE_TYPE_ATOMIC) if info.key?(:provision)
+
     info[:reconfigure][:fqname] ||= default_reconfiguration_entry_point if info.key?(:reconfigure)
 
-    info[:retirement] ||= {}
+    # By default, for retirement(terraform destroy) will have same config as provision config,
+    # because retirement(terraform destroy) action is run in-reverse order with same terraform template.
+    info[:retirement] ||= retirement_default_info(info[:provision])
     info[:retirement][:fqname] ||= default_retirement_entry_point
 
     raise _("Must provide a configuration_script_payload_id") if info[:provision][:configuration_script_payload_id].nil?
 
     info
   end
-  private_class_method :validate_config_info
+
+  private_class_method def self.retirement_default_info(prov_info)
+    info = prov_info.deep_dup
+    info.delete(:fqname)
+    info
+  end
 
   def terraform_template(action)
     template_id = config_info.dig(action.downcase.to_sym, :configuration_script_payload_id)

app/models/service_terraform_template.rb

@@ -57,6 +57,7 @@ def check_completed(action)
   def launch_terraform_template(action)
     terraform_template = terraform_template(action)
 
+    # runs provision or retirement job, based on job_options
     stack = ManageIQ::Providers::EmbeddedTerraform::AutomationManager::Stack.create_stack(terraform_template, get_job_options(action))
     add_resource!(stack, :name => action)
   end
@@ -75,8 +76,30 @@ def check_refreshed(_action)
 
   private
 
+  def job(action)
+    stack(action)&.miq_task&.job
+  end
+
   def get_job_options(action)
-    options[job_option_key(action)].deep_dup
+    job_options = options[job_option_key(action)].deep_dup
+
+    if action == ResourceAction::RETIREMENT
+      prov_job = job(ResourceAction::PROVISION)
+      if prov_job.present? && prov_job.options.present?
+        # Copy input-vars from Provision(terraform apply) action,
+        # the Retirement(terraform destroy) action will use same input-vars/values.
+        prov_vars = prov_job.options.dig(:input_vars, :extra_vars)
+        job_options[:extra_vars] = prov_vars.deep_merge!(job_options[:extra_vars]) if prov_vars
+
+        # add stack_id for terraform-runner
+        job_options[:terraform_stack_id] = prov_job.options[:terraform_stack_id]
+      end
+    end
+
+    # current action, required to identify Retirement action
+    job_options[:action] = action
+
+    job_options
   end
 
   def config_options(action)

lib/terraform/runner.rb

@@ -15,33 +15,58 @@ def available?
         @available = false
       end
 
-      # Run a template, initiates terraform-runner job for running a template, via terraform-runner api
+      # Provision or Create (terraform apply) stack in terraform-runner from a terraform template.
       #
-      # @param input_vars [Hash] Hash with key/value pairs that will be passed as input variables to the
-      #        terraform-runner run
-      # @param template_path [String] Path to the template we will want to run
-      # @param tags [Hash] Hash with key/values pairs that will be passed as tags to the terraform-runner run
-      # @param credentials [Array] List of Authentication objects to provide to the terraform run
-      # @param env_vars [Hash] Hash with key/value pairs that will be passed as environment variables to the
-      #        terraform-runner run
-      # @return [Terraform::Runner::ResponseAsync] Response object of terraform-runner create action
-      def run_async(input_vars, template_path, tags: nil, credentials: [], env_vars: {})
-        _log.debug("Run_aysnc template: #{template_path}")
-        response = create_stack_job(
-          template_path,
-          :input_vars  => input_vars,
-          :tags        => tags,
-          :credentials => credentials,
-          :env_vars    => env_vars
-        )
-        Terraform::Runner::ResponseAsync.new(response.stack_id)
+      # @param template_path [String] (required) path to the terraform template directory.
+      # @param input_vars    [Hash]   (optional) key/value pairs as input variables for the terraform-runner run job.
+      # @param tags          [Hash]   (optional) key/value pairs tags for terraform-runner Provisioned resources.
+      # @param credentials   [Array]  (optional) List of Authentication objects for the terraform run job.
+      # @param env_vars      [Hash]   (optional) key/value pairs used as environment variables, for terraform-runner run job.
+      #
+      # @return [Terraform::Runner::ResponseAsync] Response object of terraform-runner api call
+      def create_stack(template_path, input_vars: {}, tags: nil, credentials: [], env_vars: {})
+        _log.debug("Run_aysnc/create_stack for template: #{template_path}")
+        if template_path.present?
+          response = create_stack_job(
+            template_path,
+            :input_vars  => input_vars,
+            :tags        => tags,
+            :credentials => credentials,
+            :env_vars    => env_vars
+          )
+          Terraform::Runner::ResponseAsync.new(response.stack_id)
+        else
+          raise "'template_path' is required for #{ResourceAction::Provision} action"
+        end
       end
 
-      # To simplify clients who may just call run, we alias it to call
-      # run_async.  If we ever need run_sync, we'll need to revisit this.
-      alias run run_async
+      # Retire or Delete(terraform destroy) the terraform-runner created stack resources.
+      #
+      # @param stack_id      [String] (optional) required, if running ResourceAction::RETIREMENT action, used by Terraform-Runner stack_delete job.
+      # @param template_path [String] (required) path to the terraform template directory.
+      # @param input_vars    [Hash]   (optional) key/value pairs as input variables for the terraform-runner run job.
+      # @param credentials   [Array]  (optional) List of Authentication objects for the terraform run job.
+      # @param env_vars      [Hash]   (optional) key/value pairs used as environment variables, for terraform-runner run job.
+      #
+      # @return [Terraform::Runner::ResponseAsync] Response object of terraform-runner api call
+      def delete_stack(stack_id, template_path, input_vars: {}, credentials: [], env_vars: {})
+        if stack_id.present? && template_path.present?
+          _log.debug("Run_aysnc/delete_stack('#{stack_id}') for template: #{template_path}")
+          response = delete_stack_job(
+            stack_id,
+            template_path,
+            :input_vars  => input_vars,
+            :credentials => credentials,
+            :env_vars    => env_vars
+          )
+          Terraform::Runner::ResponseAsync.new(response.stack_id)
+        else
+          _log.error("'stack_id' && 'template_path' are required for #{ResourceAction::RETIREMENT} action")
+          raise "'stack_id' && 'template_path' are required for #{ResourceAction::RETIREMENT} action"
+        end
+      end
 
-      # Stop running terraform-runner job by stack_id
+      # Stop running terraform-runner job, by stack_id
       #
       # @param stack_id [String] stack_id from the terraforn-runner job
       #
@@ -50,15 +75,21 @@ def stop_async(stack_id)
         cancel_stack_job(stack_id)
       end
 
-      # Fetch terraform-runner job result/status by stack_id
+      # To simplify clients who want to stop a running stack job, we alias it to call stop_async
+      alias stop_stack stop_async
+
+      # Fetch stack object(with result/status), by stack_id from terraform-runner
       #
-      # @param stack_id [String] stack_id from the terraforn-runner job
+      # @param stack_id [String] stack_id for the terraforn-runner stack job
       #
       # @return [Terraform::Runner::Response] Response object with result of terraform run
       def fetch_result_by_stack_id(stack_id)
         retrieve_stack_job(stack_id)
       end
 
+      # To simplify clients who want to fetch stack object from terraform-runner
+      alias stack fetch_result_by_stack_id
+
       # Parse Terraform Template input/output variables
       # @param template_path [String] Path to the template we will want to parse for input/output variables
       # @return Response(body) object of terraform-runner api/template/variables,
@@ -122,7 +153,7 @@ def provider_connection_parameters(credentials)
       # Create TerraformRunner Stack Job
       def create_stack_job(
         template_path,
-        input_vars: [],
+        input_vars: {},
         tags: nil,
         credentials: [],
         env_vars: {},
@@ -150,6 +181,37 @@ def create_stack_job(
         Terraform::Runner::Response.parsed_response(http_response)
       end
 
+      # Delete(destroy) stack created by TerraformRunner Stack Job
+      def delete_stack_job(
+        stack_id,
+        template_path,
+        input_vars: {},
+        credentials: [],
+        env_vars: {}
+      )
+        _log.info("start stack_job for template: #{template_path}")
+        tenant_id = stack_tenant_id
+        encoded_zip_file = encoded_zip_from_directory(template_path)
+
+        # TODO: use tags,env_vars
+        payload = {
+          :stack_id        => stack_id,
+          :cloud_providers => provider_connection_parameters(credentials),
+          :name            => name,
+          :tenantId        => tenant_id,
+          :templateZipFile => encoded_zip_file,
+          :parameters      => ApiParams.to_cam_parameters(input_vars)
+        }
+
+        http_response = terraform_runner_client.post(
+          "api/stack/delete",
+          *json_post_arguments(payload)
+        )
+        _log.debug("==== http_response.body: \n #{http_response.body}")
+        _log.info("stack_job for template: #{template_path} running ...")
+        Terraform::Runner::Response.parsed_response(http_response)
+      end
+
       # Retrieve TerraformRunner Stack Job details
       def retrieve_stack_job(stack_id)
         http_response = terraform_runner_client.post(

lib/terraform/runner/response_async.rb

@@ -26,7 +26,7 @@ def running?
       def stop
         raise "No job running to stop" if !running?
 
-        Terraform::Runner.stop_async(@stack_id)
+        Terraform::Runner.stop_stack(@stack_id)
       end
 
       # Re-Fetch async job's response

spec/lib/terraform/runner/data/responses/hello-world-delete-in-progress.json

@@ -0,0 +1,37 @@
+{
+    "stack_id": "72259c10-e84c-11ee-a474-2d77bed8e73a",
+    "stack_job_id": 40,
+    "stack_job_is_latest": true,
+    "status": "IN_PROGRESS",
+    "action": "DELETE",
+    "message": "Wed Sep 04 2024 08:49:09 GMT+0530 (India Standard Time) Running terraform init ... \n\n\u001b[0m\u001b[1mInitializing provider plugins...\u001b[0m\n- Finding latest version of hashicorp/null...\n",
+    "error_message": "",
+    "details": {
+        "resources": [
+            {
+                "name": "null_resource.null_resource_simple",
+                "consolelinks": [],
+                "details": {
+                    "id": "4731093289508954906",
+                    "triggers": null
+                },
+                "tainted": false,
+                "idFromProvider": "4731093289508954906",
+                "typeFromProvider": "null_resource",
+                "type": "unknown",
+                "provider": "unknown"
+            }
+        ],
+        "outputs": [
+            {
+                "name": "greeting",
+                "type": "string",
+                "value": "Hello Mumbai"
+            }
+        ]
+    },
+    "stack_name": "Terraform::Runner",
+    "created_at": "2024-09-04T03:19:09.592Z",
+    "metadata": null,
+    "stack_job_start_time": "2024-09-04T03:19:09.592Z"
+}

spec/lib/terraform/runner/data/responses/hello-world-delete-success.json

@@ -0,0 +1,15 @@
+{
+    "stack_id": "72259c10-e84c-11ee-a474-2d77bed8e73a",
+    "stack_job_id": 40,
+    "stack_job_is_latest": true,
+    "status": "SUCCESS",
+    "action": "DELETE",
+    "message": "Wed Sep 04 2024 08:49:09 GMT+0530 (India Standard Time) Running terraform init ... \n\n[0m[1mInitializing provider plugins...[0m\n- Finding latest version of hashicorp/null...\n- Installing hashicorp/null v3.2.2...\n- Installed hashicorp/null v3.2.2 (signed, key ID [0m[1m0C0AF313E5FD9F80[0m[0m)\n\nProviders are signed by their developers.\nIf you'd like to know more about provider signing, you can read about it here:\nhttps://opentofu.org/docs/cli/plugins/signing/\n\nOpenTofu has created a lock file [1m.terraform.lock.hcl[0m to record the provider\nselections it made above. Include this file in your version control repository\nso that OpenTofu can guarantee to make the same selections by default when\nyou run \"tofu init\" in the future.[0m\n\n[0m[1m[32mOpenTofu has been successfully initialized![0m[32m[0m\n[0m[32m\nYou may now begin working with OpenTofu. Try running \"tofu plan\" to see\nany changes that are required for your infrastructure. All OpenTofu commands\nshould now work.\n\nIf you ever set or change modules or backend configuration for OpenTofu,\nrerun this command to reinitialize your working directory. If you forget, other\ncommands will detect it and remind you to do so if necessary.[0m\n\n\nWed Sep 04 2024 08:49:14 GMT+0530 (India Standard Time) Running terraform destroy ... \n[0m[1mnull_resource.null_resource_simple: Refreshing state... [id=4731093289508954906][0m\n\nOpenTofu used the selected providers to generate the following execution\nplan. Resource actions are indicated with the following symbols:\n  [31m-[0m destroy[0m\n\nOpenTofu will perform the following actions:\n\n[1m  # null_resource.null_resource_simple[0m will be [1m[31mdestroyed[0m\n[0m  [31m-[0m[0m resource \"null_resource\" \"null_resource_simple\" {\n      [31m-[0m[0m id = \"4731093289508954906\" [90m-> null[0m[0m\n    }\n\n[1mPlan:[0m 0 to add, 0 to change, 1 to destroy.\n[0m\nChanges to Outputs:\n  [31m-[0m[0m greeting = \"Hello Mumbai\" [90m-> null[0m[0m\n[0m[1mnull_resource.null_resource_simple: Destroying... [id=4731093289508954906][0m[0m\n[0m[1mnull_resource.null_resource_simple: Destruction complete after 0s[0m\n[0m[1m[32m\nDestroy complete! Resources: 1 destroyed.\n[0m",
+    "error_message": "",
+    "details": {"resources":[], "outputs":[]},
+    "stack_name": "Terraform::Runner",
+    "created_at": "2024-09-04T03:19:09.592Z",
+    "metadata": null,
+    "stack_job_start_time": "2024-09-04T03:19:09.592Z",
+    "stack_job_end_time": "2024-09-04T03:19:15.620Z"
+}