cleanup log-based alerts

ChainSafe · Jan 18, 2024 · 84ed14a · 84ed14a
1 parent 171f094
commit 84ed14a
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 120 deletions.
diff --git a/tf-managed/modules/daily-snapshot/monitoring/event_rules.tf b/tf-managed/modules/daily-snapshot/monitoring/event_rules.tf
diff --git a/tf-managed/modules/daily-snapshot/monitoring/main.tf b/tf-managed/modules/daily-snapshot/monitoring/main.tf
@@ -6,33 +6,31 @@ locals {
   enable_email = var.alert_email != ""
 }
 
-# resource "newrelic_nrql_alert_condition" "disk_space" {
-#   policy_id                    = newrelic_alert_policy.alert.id
-#   type                         = "static"
-#   name                         = "High Disk Utilization"
-#   description                  = "Alert when disk space usage is high on an the service host"
-#   enabled                      = true
-#
-#   nrql {
-#     query = "SELECT latest(diskUsedPercent) FROM StorageSample where entityName = '${var.service_name}'"
-#   }
-#
-#   critical {
-#     operator              = "above"
-#     # threshold             = 85.0
-#     threshold             = 20.0
-#     threshold_duration    = 300
-#     threshold_occurrences = "ALL"
-#   }
-#
-#   warning {
-#     operator              = "above"
-#     # threshold             = 70.0
-#     threshold             = 10.0
-#     threshold_duration    = 300
-#     threshold_occurrences = "ALL"
-#   }
-# }
+resource "newrelic_nrql_alert_condition" "disk_space" {
+  policy_id                    = newrelic_alert_policy.alert.id
+  type                         = "static"
+  name                         = "High Disk Utilization"
+  description                  = "Alert when disk space usage is high on an the service host"
+  enabled                      = true
+
+  nrql {
+    query = "SELECT latest(diskUsedPercent) FROM StorageSample where entityName = '${var.service_name}'"
+  }
+
+  critical {
+    operator              = "above"
+    threshold             = 85.0
+    threshold_duration    = 300
+    threshold_occurrences = "ALL"
+  }
+
+  warning {
+    operator              = "above"
+    threshold             = 70.0
+    threshold_duration    = 300
+    threshold_occurrences = "ALL"
+  }
+}
 
 resource "newrelic_notification_destination" "email" {
   count = local.enable_email ? 1 : 0
@@ -120,68 +118,3 @@ resource "newrelic_workflow" "alerting-workflow-slack" {
     channel_id = newrelic_notification_channel.slack-channel[0].id
   }
 }
-
-# At least 1 snapshot is generated in 5 hours interval
-resource "newrelic_nrql_alert_condition" "snapshot_frequency_condition" {
-  for_each    = toset(["mainnet", "calibnet"])
-  policy_id   = newrelic_alert_policy.alert.id
-  type        = "static"
-  name        = format("Low snapshot generation frequency - %s", each.key)
-  description = "Alert when snapshots are not generated within requried time interval"
-  enabled     = true
-
-  # evaluation_delay = 7200 # 2 hours, it may take some time to generate a snapshot
-  # aggregation_window = 14400 # 4 hours, it may take some time to generate a snapshot
-  aggregation_window = 360 # 4 hours, it may take some time to generate a snapshot
-
-
-  nrql {
-    query = format("FROM Metric SELECT count(`${var.service_name}.${each.key}.snapshot_generation_ok`)")
-  }
-
-  warning {
-    operator  = "below"
-    threshold = 1
-    # threshold_duration    = 14400
-    threshold_duration    = 360
-    threshold_occurrences = "ALL"
-  }
-
-  critical {
-    operator  = "below"
-    threshold = 1
-    # threshold_duration    = 28800
-    threshold_duration    = 720
-    threshold_occurrences = "ALL"
-  }
-}
-
-# At least 1 successful snapshot out of 3 attempts
-
-#resource "newrelic_nrql_alert_condition" "disk_space" {
-#  policy_id                    = newrelic_alert_policy.alert.id
-#  type                         = "static"
-#  name                         = "High Disk Utilization"
-#  description                  = "Alert when disk space usage is high on an the service host"
-#  enabled                      = true
-#
-#  nrql {
-#    query = "SELECT latest(diskUsedPercent) FROM StorageSample where entityName = '${var.service_name}'"
-#  }
-#
-#  critical {
-#    operator              = "above"
-#    # threshold             = 85.0
-#    threshold             = 20.0
-#    threshold_duration    = 300
-#    threshold_occurrences = "ALL"
-#  }
-#
-#  warning {
-#    operator              = "above"
-#    # threshold             = 70.0
-#    threshold             = 10.0
-#    threshold_duration    = 300
-#    threshold_occurrences = "ALL"
-#  }
-#}