Log based alerts is useful when you want to alert your team whenever certain logs occurs or some error or exception breaks some functionality of your application.
This guide will help you to setup the log based alerts in GCP project
Prerequisite
- GCP Project ID
- Log filter or Queries to setup alerts
- Notification Channels setuped in GCP to send alerts. (Slack or Pagerduty or email)
- Markdown Doc to send with alerts.
Add Logbased Alerts
First convert your markdown doc into a string.
sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g' path/to/file.md
This string will be used in content
field of alerts configs.
Step 1
- Open alerts_config.auto.tfvars file
- Add alerts configs map object below the commented object.
email_alert_policy_configs = [ # { # alert_display_name = "Your email Alert Name" # filter = "log queries filters" # notification_channels = ["Display name of Email notification channels"] # condition_display_name = "Name of the Condition" # alert_strategy = { # period = "300s" # auto_close = "1800s" # } # content = "Markdown file in string. Read README.md for convert file into String" # } ]
Setup terraform module
File directory structure
.
├── alerts
│ ├── alerts.tf
│ └── variables.tf
├── alerts_config.auto.tfvars
├── .gitignore
├── main.tf
├── README.md
└── variables.tf
main.tf
terraform {
required_providers {
google = {
source = "hashicorp/google"
version = "4.20.0"
}
}
}
module "log-based-alerts" {
slack_alert_policy_configs = var.slack_alert_policy_configs
pagerduty_alert_policy_configs = var.pagerduty_alert_policy_configs
email_alert_policy_configs = var.email_alert_policy_configs
source = "./alerts"
}
alerts_config.auto.tfvars
slack_alert_policy_configs = [
# {
# alert_display_name = "Your Slack Alert Name"
# filter = "log queries filters"
# notification_channels = ["Display name of Slack notification channels"]
# condition_display_name = "Name of the Condition"
# alert_strategy = {
# period = "300s"
# auto_close = "1800s"
# }
# content = "Markdown file in string. Read README.md for convert file into String"
# }
]
pagerduty_alert_policy_configs = [
# {
# alert_display_name = "Your Pageduty Alert Name"
# filter = "log queries filters"
# notification_channels = ["Display name of Pagerduty notification channels"]
# condition_display_name = "Name of the Condition"
# alert_strategy = {
# period = "300s"
# auto_close = "1800s"
# }
# content = "Markdown file in string. Read README.md for convert file into String"
# }
]
email_alert_policy_configs = [
# {
# alert_display_name = “Your email Alert Name”
# filter = “log queries filters”
# notification_channels = [“Display name of Email notification channels”]
# condition_display_name = “Name of the Condition”
# alert_strategy = {
# period = “300s”
# auto_close = “1800s”
# }
# content = “Markdown file in string. Read README.md for convert file into String”
# }
]
variables.tf
variable "slack_alert_policy_configs" {
}
variable "pagerduty_alert_policy_configs" {
}
variable "email_alert_policy_configs" {
}
Setup Alert module in alert/ directory
alerts.tf
locals {
log_based_slack_alerts_configs ={
for alc in var.slack_alert_policy_configs :
alc.alert_display_name => {
filter = alc.filter
notification_channels = alc.notification_channels
condition_display_name = alc.condition_display_name
alert_strategy = alc.alert_strategy
content = alc.content
}
}
log_based_pagerduty_alerts_configs={
for alc in var.pagerduty_alert_policy_configs :
alc.alert_display_name => {
filter = alc.filter
notification_channels = alc.notification_channels
condition_display_name = alc.condition_display_name
alert_strategy = alc.alert_strategy
content = alc.content
}
}
log_based_email_alerts_configs={
for alc in var.email_alert_policy_configs :
alc.alert_display_name => {
filter = alc.filter
notification_channels = alc.notification_channels
condition_display_name = alc.condition_display_name
alert_strategy = alc.alert_strategy
content = alc.content
}
}
}
resource "google_monitoring_alert_policy" "slack_alert_policy" {
for_each = local.log_based_slack_alerts_configs
display_name = each.key
documentation {
content = each.value.content
mime_type = "text/markdown"
}
enabled = true
alert_strategy {
notification_rate_limit {
period = each.value.alert_strategy.period
}
auto_close = each.value.alert_strategy.auto_close
}
notification_channels = data.google_monitoring_notification_channel.slack_channels[*].id
combiner = "OR"
conditions {
display_name = each.value.condition_display_name
condition_matched_log {
filter = each.value.filter
}
}
}
resource "google_monitoring_alert_policy" "pagerduty_alert_policy" {
for_each = local.log_based_pagerduty_alerts_configs
display_name = each.key
documentation {
content = each.value.content
mime_type = "text/markdown"
}
enabled = true
alert_strategy {
notification_rate_limit {
period = each.value.alert_strategy.period
}
auto_close = each.value.alert_strategy.auto_close
}
notification_channels = data.google_monitoring_notification_channel.pagerduty_channels[*].id
combiner = "OR"
conditions {
display_name = each.value.condition_display_name
condition_matched_log {Install provider plugins
terraform init
Check Terraform plan
terraform plan
Apply the terraform configurations
terraform apply
filter = each.value.filter
}
}
}
resource "google_monitoring_alert_policy" "email_alert_policy" {
for_each = local.log_based_email_alerts_configs
display_name = each.key
documentation {
content = each.value.content
mime_type = "text/markdown"
}
enabled = true
alert_strategy {
notification_rate_limit {
period = each.value.alert_strategy.period
}
auto_close = each.value.alert_strategy.auto_close
}
notification_channels = data.google_monitoring_notification_channel.email_channels[*].id
combiner = "OR"
conditions {
display_name = each.value.condition_display_name
condition_matched_log {
filter = each.value.filter
}
}
}
data "google_monitoring_notification_channel" "slack_channels" {
count = length(var.slack_alert_policy_configs) > 0 ?length(var.slack_alert_policy_configs[0].notification_channels):0
display_name = var.slack_alert_policy_configs[0].notification_channels[count.index]
}
data "google_monitoring_notification_channel" "pagerduty_channels" {
count = length(var.pagerduty_alert_policy_configs) > 0 ?length(var.pagerduty_alert_policy_configs[0].notification_channels):0
display_name = var.pagerduty_alert_policy_configs[0].notification_channels[count.index]
}
data "google_monitoring_notification_channel" "email_channels" {
count = length(var.email_alert_policy_configs) > 0 ?length(var.email_alert_policy_configs[0].notification_channels):0
display_name = var.email_alert_policy_configs[0].notification_channels[count.index]
}
variables.tf
variable "slack_alert_policy_configs" {
default = []
}
variable "pagerduty_alert_policy_configs" {
default = []
}
variable "email_alert_policy_configs" {
default = []
}
Usage
- Install provider plugins
terraform init
- Check Terraform plan
terraform plan
- Apply the terraform configurations
terraform apply
