Setup eks connections to the database (#87)

* setup eks connections * setup formatting and docs generation scripts * setup tefca jdbc database connection strings and update db url name * modified terraform to add changes identified in rds-ssl-fix.md file * modified enabled_cloudwatch_logs_exports * fixed formatting --------- Co-authored-by: shanice-skylight <shanice@skylight.digital>
CDCgov · Oct 18, 2024 · 6570493 · 6570493
1 parent 39f589d
commit 6570493
Show file tree

Hide file tree

Showing 23 changed files with 650 additions and 38 deletions.
diff --git a/terraform/aws/docs/log_error.png b/terraform/aws/docs/log_error.png
diff --git a/terraform/aws/docs/log_fix.png b/terraform/aws/docs/log_fix.png
diff --git a/terraform/aws/docs/modified_iam_policy.png b/terraform/aws/docs/modified_iam_policy.png
diff --git a/terraform/aws/docs/original_iam_policy.png b/terraform/aws/docs/original_iam_policy.png
diff --git a/terraform/aws/docs/rds-ssl-fix.md b/terraform/aws/docs/rds-ssl-fix.md
@@ -0,0 +1,26 @@
+There were multiple configurations that prohibited the EKS tefca viewer pod from connecting to the Postgres database. 
+
+1. The EKS clusterwas missing the required IAM permissions (add to phdi-playground repo)
+    1. 
+    - Original policy had  `rds:Connect` as an action it was creating the error `Invalid Action: The action rds:Connect does not exist`
+    ![Original Policy](./original_iam_policy.png) 
+
+    - Modified Policy : This policy replaced rds:Connect with rds-db:Connect and the resource block to reference the database with the Resource ID
+    ![Modified Policy](./modified_iam_policy.png) 
+
+
+2. In the parameter group for the database, rds.force_ssl was enabled and only allows SSL connections.
+    - I disable the parameter by changing the value to 0
+
+    The fix is sufficient to allow the connection between the pod and RDS.
+
+    - Log with error message:
+    ![Log error](./log_error.png)
+
+    - Log after making changes above:
+        - In the snapshot, I can verify that the database is connecting based off of the 3 entries that show connection received, connection authenticated, and connection authorized
+    ![Log fix](./log_fix.png)
+
+3. To better assist with troubleshooting in the future, I made the change below:
+    -  I turned on Postgres logging for the RDS database to help with debugging the issue
+
diff --git a/terraform/aws/implementation/README.md b/terraform/aws/implementation/README.md
@@ -0,0 +1,43 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | =5.61.0 |
+| <a name="requirement_external"></a> [external](#requirement\_external) | = 2.3.3 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | = 2.12.1 |
+| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | = 2.25.2 |
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_cognito"></a> [cognito](#module\_cognito) | ./modules/cognito | n/a |
+| <a name="module_eks"></a> [eks](#module\_eks) | ./modules/eks | n/a |
+| <a name="module_rds"></a> [rds](#module\_rds) | ./modules/rds | n/a |
+| <a name="module_route53"></a> [route53](#module\_route53) | ./modules/route53 | n/a |
+| <a name="module_s3"></a> [s3](#module\_s3) | ./modules/s3 | n/a |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_enable_cognito"></a> [enable\_cognito](#input\_enable\_cognito) | Enable Cognito | `bool` | `true` | no |
+| <a name="input_region"></a> [region](#input\_region) | AWS region | `string` | `"us-east-1"` | no |
+| <a name="input_smarty_auth_id"></a> [smarty\_auth\_id](#input\_smarty\_auth\_id) | value of the SmartyStreets Auth ID | `any` | n/a | yes |
+| <a name="input_smarty_auth_token"></a> [smarty\_auth\_token](#input\_smarty\_auth\_token) | value of the SmartyStreets Auth Token | `any` | n/a | yes |
+
+## Outputs
+
+No outputs.
+<!-- END_TF_DOCS -->
diff --git a/terraform/aws/implementation/main.tf b/terraform/aws/implementation/main.tf
@@ -31,23 +31,27 @@ module "vpc" {
 }
 
 module "eks" {
-  source                   = "./modules/eks"
-  region                   = var.region
-  eks_name                 = local.name
-  vpc_id                   = module.vpc.vpc_id
-  public_subnet_ids        = module.vpc.public_subnets
-  private_subnet_ids       = module.vpc.private_subnets
-  smarty_auth_id           = var.smarty_auth_id
-  smarty_auth_token        = var.smarty_auth_token
-  aws_acm_certificate_arn  = module.route53.aws_acm_certificate_arn
-  ecr_viewer_s3_role_arn   = module.s3.ecr_viewer_s3_role_arn
-  tefca_viewer_db_role_arn = module.rds.tefca_viewer_db_role_arn
-  domain_name              = local.domain_name
-  ecr_bucket_name          = module.s3.ecr_bucket_name
-  enable_cognito           = var.enable_cognito
-  cognito_user_pool_arn    = module.cognito.cognito_user_pool_arn
-  cognito_client_id        = module.cognito.cognito_client_id
-  cognito_domain           = module.cognito.cognito_domain
+  source                     = "./modules/eks"
+  region                     = var.region
+  eks_name                   = local.name
+  vpc_id                     = module.vpc.vpc_id
+  public_subnet_ids          = module.vpc.public_subnets
+  private_subnet_ids         = module.vpc.private_subnets
+  smarty_auth_id             = var.smarty_auth_id
+  smarty_auth_token          = var.smarty_auth_token
+  aws_acm_certificate_arn    = module.route53.aws_acm_certificate_arn
+  ecr_viewer_s3_role_arn     = module.s3.ecr_viewer_s3_role_arn
+  tefca_viewer_db_role_arn   = module.rds.tefca_viewer_db_role_arn
+  tefca_db_connection_string = module.rds.tefca_db_connection_string
+  tefca_jdbc_db_url          = module.rds.tefca_jdbc_db_url
+  tefca_jdbc_db_password     = module.rds.tefca_jdbc_db_password
+  tefca_jdbc_db_user         = module.rds.tefca_jdbc_db_user
+  domain_name                = local.domain_name
+  ecr_bucket_name            = module.s3.ecr_bucket_name
+  enable_cognito             = var.enable_cognito
+  cognito_user_pool_arn      = module.cognito.cognito_user_pool_arn
+  cognito_client_id          = module.cognito.cognito_client_id
+  cognito_domain             = module.cognito.cognito_domain
 }
 
 module "route53" {

diff --git a/terraform/aws/implementation/modules/cognito/README.md b/terraform/aws/implementation/modules/cognito/README.md
@@ -0,0 +1,39 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cognito_user.admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cognito_user) | resource |
+| [aws_cognito_user.dibbs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cognito_user) | resource |
+| [aws_cognito_user_pool.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cognito_user_pool) | resource |
+| [aws_cognito_user_pool_client.client](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cognito_user_pool_client) | resource |
+| [aws_cognito_user_pool_domain.domain](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cognito_user_pool_domain) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_domain_name"></a> [domain\_name](#input\_domain\_name) | The domain name for ALB | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_cognito_client_id"></a> [cognito\_client\_id](#output\_cognito\_client\_id) | n/a |
+| <a name="output_cognito_domain"></a> [cognito\_domain](#output\_cognito\_domain) | n/a |
+| <a name="output_cognito_user_pool_arn"></a> [cognito\_user\_pool\_arn](#output\_cognito\_user\_pool\_arn) | n/a |
+<!-- END_TF_DOCS -->
diff --git a/terraform/aws/implementation/modules/eks/README.md b/terraform/aws/implementation/modules/eks/README.md
@@ -0,0 +1,94 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+| <a name="provider_external"></a> [external](#provider\_external) | n/a |
+| <a name="provider_helm"></a> [helm](#provider\_helm) | n/a |
+| <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | >= 1.14.0 |
+| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | n/a |
+| <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_eks-cluster"></a> [eks-cluster](#module\_eks-cluster) | terraform-aws-modules/eks/aws | 19.21.0 |
+| <a name="module_eks_blueprints_addons"></a> [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.14 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_policy.cloudwatch_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.eks_service_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [helm_release.building_blocks](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubectl_manifest.cluster_role](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.cluster_role_binding](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.ingress](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.load_balancer_controller_crds](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.load_balancer_service_account](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.logging_config_map](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubernetes_namespace_v1.aws_observability](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
+| [terraform_data.helm_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
+| [terraform_data.kubeconfig](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
+| [terraform_data.wait_for_load_balancer_controller](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source |
+| [aws_eks_cluster_auth.eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
+| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.cloudwatch_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.eks_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [external_external.chart_versions](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source |
+| [external_external.latest_phdi_release](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source |
+| [kubectl_file_documents.ingress](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/file_documents) | data source |
+| [kubectl_file_documents.load_balancer_controller_crds](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/file_documents) | data source |
+| [kubectl_file_documents.load_balancer_service_account](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/file_documents) | data source |
+| [kubectl_file_documents.logging_config_map](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/file_documents) | data source |
+| [kubectl_path_documents.cluster_role](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source |
+| [kubectl_path_documents.cluster_role_binding](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_aws_acm_certificate_arn"></a> [aws\_acm\_certificate\_arn](#input\_aws\_acm\_certificate\_arn) | The ARN of the ACM certificate | `any` | n/a | yes |
+| <a name="input_cognito_client_id"></a> [cognito\_client\_id](#input\_cognito\_client\_id) | The ID of the Cognito user pool client | `any` | n/a | yes |
+| <a name="input_cognito_domain"></a> [cognito\_domain](#input\_cognito\_domain) | The domain of the Cognito user pool | `any` | n/a | yes |
+| <a name="input_cognito_user_pool_arn"></a> [cognito\_user\_pool\_arn](#input\_cognito\_user\_pool\_arn) | The ARN of the Cognito user pool | `any` | n/a | yes |
+| <a name="input_domain_name"></a> [domain\_name](#input\_domain\_name) | The domain name to use | `string` | n/a | yes |
+| <a name="input_ecr_bucket_name"></a> [ecr\_bucket\_name](#input\_ecr\_bucket\_name) | The name of the ECR bucket | `string` | n/a | yes |
+| <a name="input_ecr_viewer_s3_role_arn"></a> [ecr\_viewer\_s3\_role\_arn](#input\_ecr\_viewer\_s3\_role\_arn) | The s3 Role ARN for the ECR Viewer Service | `any` | n/a | yes |
+| <a name="input_eks_name"></a> [eks\_name](#input\_eks\_name) | n/a | `string` | `"phdi-playground-eks"` | no |
+| <a name="input_enable_cognito"></a> [enable\_cognito](#input\_enable\_cognito) | Enable Cognito | `bool` | `true` | no |
+| <a name="input_private_subnet_ids"></a> [private\_subnet\_ids](#input\_private\_subnet\_ids) | List of private subnet IDs | `list(string)` | n/a | yes |
+| <a name="input_public_subnet_ids"></a> [public\_subnet\_ids](#input\_public\_subnet\_ids) | List of public subnet IDs | `list(string)` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | n/a | `string` | `"us-east-1"` | no |
+| <a name="input_services_to_chart"></a> [services\_to\_chart](#input\_services\_to\_chart) | Note: The chart names are limited to 15 characters | `map(string)` | <pre>{<br>  "ecr-viewer": "ecr-viewer",<br>  "fhir-converter": "fhir-converter",<br>  "ingestion": "ingestion",<br>  "message-parser": "message-parser",<br>  "message-refiner": "message-refiner",<br>  "orchestration": "orchestration",<br>  "tefca-viewer": "tefca-viewer",<br>  "trigger-code-reference": "trigger-code-reference",<br>  "validation": "validation"<br>}</pre> | no |
+| <a name="input_smarty_auth_id"></a> [smarty\_auth\_id](#input\_smarty\_auth\_id) | value of the SmartyStreets Auth ID | `any` | n/a | yes |
+| <a name="input_smarty_auth_token"></a> [smarty\_auth\_token](#input\_smarty\_auth\_token) | value of the SmartyStreets Auth Token | `any` | n/a | yes |
+| <a name="input_tefca_db_connection_string"></a> [tefca\_db\_connection\_string](#input\_tefca\_db\_connection\_string) | Connection string to the tefca database | `any` | n/a | yes |
+| <a name="input_tefca_jdbc_db_password"></a> [tefca\_jdbc\_db\_password](#input\_tefca\_jdbc\_db\_password) | JDBC password for flyway to the tefca database | `any` | n/a | yes |
+| <a name="input_tefca_jdbc_db_url"></a> [tefca\_jdbc\_db\_url](#input\_tefca\_jdbc\_db\_url) | JDBC connection string for flyway to the tefca database | `any` | n/a | yes |
+| <a name="input_tefca_jdbc_db_user"></a> [tefca\_jdbc\_db\_user](#input\_tefca\_jdbc\_db\_user) | JDBC username for flyway to the tefca database | `any` | n/a | yes |
+| <a name="input_tefca_viewer_db_role_arn"></a> [tefca\_viewer\_db\_role\_arn](#input\_tefca\_viewer\_db\_role\_arn) | The db Role ARN for the Tefca Viewer Service | `any` | n/a | yes |
+| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | ID of the VPC | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_eks_assume_role_policy"></a> [eks\_assume\_role\_policy](#output\_eks\_assume\_role\_policy) | n/a |
+| <a name="output_ingress_created"></a> [ingress\_created](#output\_ingress\_created) | n/a |
+<!-- END_TF_DOCS -->
diff --git a/terraform/aws/implementation/modules/eks/main.tf b/terraform/aws/implementation/modules/eks/main.tf
@@ -306,17 +306,38 @@ resource "helm_release" "building_blocks" {
   recreate_pods   = true
   cleanup_on_fail = true
 
-  set {
-    name  = "image.tag"
-    value = data.external.latest_phdi_release.result.tagName
+  set_sensitive {
+    name  = "jdbcDatabaseUrl"
+    value = var.tefca_jdbc_db_url
+  }
+
+  set_sensitive {
+    name  = "jdbcDatabasePassword"
+    value = var.tefca_jdbc_db_password
+  }
+
+  set_sensitive {
+    name  = "jdbcDatabaseUser"
+    value = var.tefca_jdbc_db_user
+  }
+
+  set_sensitive {
+    name  = "databaseConnectionString"
+    value = var.tefca_db_connection_string
   }
 
   set {
+    name = "image.tag"
+    # value = data.external.latest_phdi_release.result.tagName
+    value = "v1.6.7"
+  }
+
+  set_sensitive {
     name  = "smartyAuthId"
     value = var.smarty_auth_id
   }
 
-  set {
+  set_sensitive {
     name  = "smartyToken"
     value = var.smarty_auth_token
   }
@@ -332,6 +353,8 @@ resource "helm_release" "building_blocks" {
   }
 
   #  Values needed for orchestration service
+  # "phdi-playground-${terraform.workspace}-${each.key}-${each.key}-service"
+  # phdi-playground-dev-ecr-viewer-ecr-viewer-service
   set {
     name  = "fhirConverterUrl"
     value = "https://${var.domain_name}/fhir-converter"

diff --git a/terraform/aws/implementation/modules/eks/variables.tf b/terraform/aws/implementation/modules/eks/variables.tf
@@ -86,3 +86,19 @@ variable "cognito_client_id" {
 variable "cognito_domain" {
   description = "The domain of the Cognito user pool"
 }
+
+variable "tefca_db_connection_string" {
+  description = "Connection string to the tefca database"
+}
+
+variable "tefca_jdbc_db_url" {
+  description = "JDBC connection string for flyway to the tefca database"
+}
+
+variable "tefca_jdbc_db_password" {
+  description = "JDBC password for flyway to the tefca database"
+}
+
+variable "tefca_jdbc_db_user" {
+  description = "JDBC username for flyway to the tefca database"
+}