Use default statefile and fix dependency breakage

With all the features of Terraform 0.12, the migrate script can
completely run off of remote state, so by default the script will let
Terraform use its default state, remote or local. There is an option to
override this for testing or for keeping remote state safe.

Add verbose option to migrate script, which will print the commands
being run even if dryrun is not enabled.

Revert the output var change in product_services, since it's needed for
the dependency graph. However, the old '.*.' operator doesn't work with
for_each, so replace it with a new [for] construct.

Author: Rycieos

docs/upgrading_to_project_factory_v4.0.md

@@ -66,40 +66,22 @@ chmod +x migrate4.py
 terraform init -upgrade
 ```
 
-### Locally download your Terraform state
+### Backup your remote Terraform state
 
 This step is only required if you are using [remote state](https://www.terraform.io/docs/state/remote.html).
 
 ```
-terraform state pull >> terraform.tfstate
-```
-
-You should then disable remote state temporarily:
-
-```hcl
-terraform {
-#  backend "gcs" {
-#    bucket  = "my-bucket-name"
-#    prefix  = "terraform/state/projects"
-#  }
-}
-```
-
-After commenting out your remote state configuration, you must re-initialize Terraform.
-
-```
-terraform init
+terraform state pull > terraform-backup.tfstate
 ```
 
 ### Migrate the Terraform state to match the new Project Factory module structure
 
 ```
-./migrate4.py terraform.tfstate terraform.tfstate.new
+./migrate4.py
 ```
 
 Expected output:
 ```txt
-cp terraform.tfstate terraform.tfstate.new
 ---- Migrating the following project-factory modules:
 -- module.my-project.module.project-factory
 Removed module.my-project.module.project-factory.google_project_service.project_services[0]
@@ -113,13 +95,13 @@ Import successful!
 
 The resources that were imported are shown above. These resources are now in
 your Terraform state and will henceforth be managed by Terraform.
-State migration complete, verify migration with `terraform plan -state 'terraform.tfstate.new'`
+State migration complete, verify migration with `terraform plan`
 ```
 
 ### Check that terraform plans for expected changes
 
 ```
-terraform plan -state terraform.tfstate.new
+terraform plan
 ```
 
 The project services refactor will show this for each service:
@@ -137,40 +119,12 @@ The project services refactor will show this for each service:
   }
 ```
 
-### Back up the old Terraform state and switch to the new Terraform state
-
-If the Terraform plan suggests no changes, replace the old state file with the new state file.
-
-```
-mv terraform.tfstate terraform.tfstate.pre-migrate
-mv terraform.tfstate.new terraform.tfstate
-```
-
 ### Run Terraform to reconcile any differences
 
 ```
 terraform apply
 ```
 
-### Re-enable remote state
-
-You should then re-enable remote state:
-
-```hcl
-terraform {
-  backend "gcs" {
-    bucket  = "my-bucket-name"
-    prefix  = "terraform/state/projects"
-  }
-}
-```
-
-After restoring remote state, you need to re-initialize Terraform and push your local state to the remote:
-
-```
-terraform init -force-copy
-```
-
 ### Clean up
 
 Once you are done with the migration, you can safely remove `migrate.py`.
@@ -179,10 +133,10 @@ Once you are done with the migration, you can safely remove `migrate.py`.
 rm migrate4.py
 ```
 
-If you are using remote state, you can also remove the local state copies.
+If you are using remote state, you can also remove the local state backup.
 
 ```
-rm -rf terraform.tfstate*
+rm -rf terraform-backup.tfstate
 ```
 
 ## Troubleshooting

helpers/migrate4.py

@@ -227,7 +227,7 @@ def read_state(statefile):
     """
     Read the terraform state at the given path.
     """
-    argv = ["terraform", "state", "list", "-state", statefile]
+    argv = ["terraform", "state", "list", *statefile]
     result = subprocess.run(argv,
                             capture_output=True,
                             check=True,
@@ -242,7 +242,7 @@ def read_resource_value(resource, field, statefile):
     Read a specific value from a resource from the terraform state at the
     given path.
     """
-    argv = ["terraform", "state", "show", "-no-color", "-state", statefile, resource]
+    argv = ["terraform", "state", "show", "-no-color", *statefile, resource]
     result = subprocess.run(argv,
                             capture_output=True,
                             check=True,
@@ -266,15 +266,15 @@ def state_changes_for_module(module, statefile):
     migration = ProjectServicesMigration(module, statefile)
 
     for (old, new, id) in migration.moves():
-        argv = ["terraform", "state", "rm", "-state", statefile, old]
+        argv = ["terraform", "state", "rm", *statefile, old]
         commands.append(argv)
-        argv = ["terraform", "import", "-state", statefile, new, id]
+        argv = ["terraform", "import", *statefile, new, id]
         commands.append(argv)
 
     return commands
 
 
-def migrate(statefile, dryrun=False):
+def migrate(statefile, dryrun=False, verbose=False):
     """
     Migrate the terraform state in `statefile` to match the post-refactor
     resource structure.
@@ -310,34 +310,38 @@ def migrate(statefile, dryrun=False):
         commands += state_changes_for_module(factory, statefile)
 
     for argv in commands:
-        if dryrun:
+        if dryrun or verbose:
             print(" ".join(argv))
-        else:
+        if not dryrun:
             subprocess.run(argv, check=True, encoding='utf-8')
 
 
 def main(argv):
     parser = argparser()
     args = parser.parse_args(argv[1:])
 
-    print("cp {} {}".format(args.oldstate, args.newstate))
-    shutil.copy(args.oldstate, args.newstate)
+    state = []
+    plan_command = "terraform plan"
+    if args.state:
+        state = ["-state", args.state]
+        plan_command += " {}".format(' '.join(state))
 
-    migrate(args.newstate, dryrun=args.dryrun)
+    migrate(state, dryrun=args.dryrun, verbose=args.verbose)
     print("State migration complete, verify migration with "
-          "`terraform plan -state '{}'`".format(args.newstate))
+          "`{}`".format(plan_command))
 
 
 def argparser():
     parser = argparse.ArgumentParser(description='Migrate Terraform state')
-    parser.add_argument('oldstate', metavar='oldstate.json',
-                        help='The current Terraform state (will not be '
-                             'modified)')
-    parser.add_argument('newstate', metavar='newstate.json',
-                        help='The path to the new state file')
+    parser.add_argument('--state',
+                        help='The Terraform state file to modify, otherwise '
+                        'Terraform default')
     parser.add_argument('--dryrun', action='store_true',
                         help='Print the `terraform state mv` commands instead '
                              'of running the commands.')
+    parser.add_argument('--verbose', '-v', action='store_true',
+                        help='Print the `terraform state mv` commands that '
+                             'are run.')
     return parser
 
 

modules/project_services/outputs.tf

@@ -16,6 +16,9 @@
 
 output "project_id" {
   description = "The GCP project you want to enable APIs on"
-  value       = var.enable_apis && length(var.activate_apis) > 0 ? var.project_id : ""
+  value = element(
+    [for v in google_project_service.project_services : v.project],
+    0,
+  )
 }