From d24cc12b795561298f059d78d77532e29e6e1275 Mon Sep 17 00:00:00 2001 From: Olamide Date: Mon, 20 Nov 2023 13:27:25 +0100 Subject: [PATCH] Add module to extract cloudwatch logs --- aws/cloudwatch-log-extract/README.md | 82 +++++++++++++ .../lambda-script/lambda_function.py | 63 ++++++++++ aws/cloudwatch-log-extract/main.tf | 111 ++++++++++++++++++ aws/cloudwatch-log-extract/makefile | 67 +++++++++++ aws/cloudwatch-log-extract/outputs.tf | 0 aws/cloudwatch-log-extract/variables.tf | 33 ++++++ aws/cloudwatch-log-extract/versions.tf | 17 +++ 7 files changed, 373 insertions(+) create mode 100644 aws/cloudwatch-log-extract/README.md create mode 100644 aws/cloudwatch-log-extract/lambda-script/lambda_function.py create mode 100644 aws/cloudwatch-log-extract/main.tf create mode 100644 aws/cloudwatch-log-extract/makefile create mode 100644 aws/cloudwatch-log-extract/outputs.tf create mode 100644 aws/cloudwatch-log-extract/variables.tf create mode 100644 aws/cloudwatch-log-extract/versions.tf diff --git a/aws/cloudwatch-log-extract/README.md b/aws/cloudwatch-log-extract/README.md new file mode 100644 index 00000000..feddedd9 --- /dev/null +++ b/aws/cloudwatch-log-extract/README.md @@ -0,0 +1,82 @@ +# CloudWatch Logs Extract Module + +This module creates a [CloudWatch logs subscription](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html#LambdaFunctionExample) to stream CloudWatch logs to a lambda function endpoint. You can pass in a Log group filter to select the Logs from the Log group that need to be streamed, you can also pass in a regex pattern to pick out important details from the log group and map each capture group to a label. +These messages will be sent as a json string to an SNS endpoint. You can pass in a message attribute detail to be added to the message sent to the SNS destination endpoint. + + +Example: + +``` +module "cloudwatch_log_extract" { + source = "github.com/thoughtbot/flightdeck//aws/cloudwatch-log-extract" + + # Enter a log group filter pattern to pick which logs get sent to the lambda endpoint + log_group_filter_pattern = "duration" + + # Enter a log message filter expression to pick out specific details to be sent to the destination SNS topic. You may provide a regex pattern with capture groups, and enter the label for each capture group. + log_message_filter = { + regex_pattern = ".* duration: ([0-9.]+) ms .* content: (.*)" + capture_group = { + 1 = "duration", + 2 = "message" + } + } + + # Enter the destination SNS topic + destination_sns_topic_arn = "arn:aws:sns:us-east-1:123456789:destination-sns-topic" + + # Enter any message attribute for the SNS message. + destination_sns_message_attributes = { + environment = "production" + } + + # Enter the CloudWatch log group to subscribe to for log messages. + source_cloudwatch_log_group = "/sample/cloudwatch/log/group" +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.8 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [aws](#requirement\_aws) | ~> 4.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [aws](#provider\_aws) | ~> 4.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.lambda_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_subscription_filter.cloudwatch_log_filter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource | +| [aws_iam_role.lambda_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.logs_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_lambda_function.sql_query_update](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.allow_cloudwatch_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [random_id.unique_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [archive_file.function](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [aws_cloudwatch_log_group.log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/cloudwatch_log_group) | data source | +| [aws_iam_policy_document.assume_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_filter_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [destination\_sns\_topic\_arn](#input\_destination\_sns\_topic\_arn) | The ARN of the destination SNS topic to deliver matching log events to. | `string` | n/a | yes | +| [log\_group\_filter\_pattern](#input\_log\_group\_filter\_pattern) | CloudWatch Logs filter pattern for subscribing to a filtered stream of log events | `string` | `""` | no | +| [log\_message\_filter](#input\_log\_message\_filter) | Filter regex pattern to pick out items from a Cloudwatch log to be sent to destination endpoint. Pass in a valid regex pattern with capture groups, and a capture\_group map stating the label for each capture\_group | 
object({
    regex_pattern = string,
    capture_group = map(string)
  })
| 
{
  "capture_group": {},
  "regex_pattern": ""
}
| no | +| [message\_attributes](#input\_message\_attributes) | Message attributes to be included with messages publised to SNS | `map(string)` | `{}` | no | +| [source\_cloudwatch\_log\_group](#input\_source\_cloudwatch\_log\_group) | The name of the log group to associate the subscription filter with | `string` | n/a | yes | + diff --git a/aws/cloudwatch-log-extract/lambda-script/lambda_function.py b/aws/cloudwatch-log-extract/lambda-script/lambda_function.py new file mode 100644 index 00000000..f5a50f0b --- /dev/null +++ b/aws/cloudwatch-log-extract/lambda-script/lambda_function.py @@ -0,0 +1,63 @@ +import json +import boto3 +import re +import os +from calendar import timegm +import gzip +import base64 + + + +def lambda_handler(event, context): + + capture_group = os.environ['capture_group'] + regex_pattern = os.environ['regex_pattern'] + message_attributes = os.environ['message_attributes'] + destinationSnsTopicArn = os.environ["destinationSnsTopicArn"] + + snsClient = boto3.client('sns') + + if capture_group: + capture_group = json.loads(capture_group) + + if message_attributes: + message_attributes = json.loads(message_attributes) + + print(f"Regex pattern is {regex_pattern}") + + print(f"Capture group is {capture_group}") + + encoded_zipped_data = event['awslogs']['data'] + zipped_data = base64.b64decode(encoded_zipped_data) + data = gzip.decompress(zipped_data) + payload = json.loads(data) + + publish_dict = {} + + for logs in payload['logEvents']: + log_message = logs["message"].replace('\t', '').replace('\n', '') + + if regex_pattern: + matchObject = re.match(regex_pattern, log_message) + if matchObject: + for key,value in capture_group.items(): + publish_dict[value] = matchObject.group(int(key)) + + else: + publish_dict["message"] = log_message + + print(f"Publish dict is {publish_dict}") + + # push message to SNS topic + response = snsClient.publish( + TopicArn=destinationSnsTopicArn, + Message=json.dumps(publish_dict), + Subject="CloudWatch-Logs-Extract", + MessageStructure='string', + MessageAttributes={ + '{}'.format(key): { + 'DataType': 'String', + 'StringValue': value, + } for key,value in message_attributes.items() + }, + ) diff --git a/aws/cloudwatch-log-extract/main.tf b/aws/cloudwatch-log-extract/main.tf new file mode 100644 index 00000000..31c068e9 --- /dev/null +++ b/aws/cloudwatch-log-extract/main.tf @@ -0,0 +1,111 @@ + +locals { + function_name = "transform-cloudwatch-log-${random_id.unique_id.dec}" +} + +resource "aws_cloudwatch_log_subscription_filter" "cloudwatch_log_filter" { + name = "cloudwatch_log_filter-${random_id.unique_id.dec}" + log_group_name = var.source_cloudwatch_log_group + filter_pattern = var.log_group_filter_pattern + destination_arn = aws_lambda_function.sql_query_update.arn +} + +resource "aws_lambda_function" "sql_query_update" { + function_name = local.function_name + description = "Lambda function to extract cloudwatch logs to be sent to SNS" + filename = data.archive_file.function.output_path + handler = "lambda_function.lambda_handler" + role = aws_iam_role.lambda_role.arn + runtime = "python3.8" + source_code_hash = data.archive_file.function.output_base64sha256 + timeout = 60 + + environment { + variables = { + capture_group = jsonencode(var.log_message_filter.capture_group) + regex_pattern = var.log_message_filter.regex_pattern + message_attributes = jsonencode(var.message_attributes) + destinationSnsTopicArn = var.destination_sns_topic_arn + } + } + depends_on = [ + aws_cloudwatch_log_group.lambda_logs, + aws_iam_role.lambda_role + ] +} + +data "aws_iam_policy_document" "assume_role_policy_doc" { + statement { + sid = "AllowAwsToAssumeRole" + effect = "Allow" + + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + + identifiers = [ + "lambda.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role" "lambda_role" { + name = "${local.function_name}-lambda-role" + assume_role_policy = data.aws_iam_policy_document.assume_role_policy_doc.json +} + +resource "aws_iam_role_policy" "logs_role_policy" { + name = "${local.function_name}-policy" + role = aws_iam_role.lambda_role.id + policy = data.aws_iam_policy_document.lambda_filter_policy.json +} + +resource "random_id" "unique_id" { + byte_length = 3 +} + +data "archive_file" "function" { + output_path = "lambda_function.zip" + source_file = "${path.module}/lambda-script/lambda_function.py" + type = "zip" +} + +data "aws_cloudwatch_log_group" "log_group" { + name = var.source_cloudwatch_log_group +} + +resource "aws_cloudwatch_log_group" "lambda_logs" { + name = "/aws/lambda/${local.function_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "lambda_filter_policy" { + statement { + sid = "snspublishnotifiations" + effect = "Allow" + resources = [var.destination_sns_topic_arn] + actions = ["sns:Publish"] + } + + statement { + sid = "cloudwatchlambdalogs" + effect = "Allow" + resources = ["arn:aws:logs:*:*:*"] + actions = [ + "logs:*" + ] + } +} + +resource "aws_lambda_permission" "allow_cloudwatch_logs" { + statement_id = "AllowExecutionFromCloudwatchLogs" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.sql_query_update.function_name + principal = "logs.${data.aws_region.current.name}.amazonaws.com" + source_arn = data.aws_cloudwatch_log_group.log_group.arn +} + +data "aws_region" "current" {} + diff --git a/aws/cloudwatch-log-extract/makefile b/aws/cloudwatch-log-extract/makefile new file mode 100644 index 00000000..364a9e25 --- /dev/null +++ b/aws/cloudwatch-log-extract/makefile @@ -0,0 +1,67 @@ +MODULEFILES := $(wildcard *.tf) +TFLINTRC ?= ../../.tflint.hcl +TFDOCSRC ?= ../../.terraform-docs.yml + +.PHONY: default +default: checkfmt validate docs lint + +.PHONY: checkfmt +checkfmt: .fmt + +.PHONY: fmt +fmt: $(MODULEFILES) + terraform fmt + @touch .fmt + +.PHONY: validate +validate: .validate + +.PHONY: docs +docs: README.md + +.PHONY: lint +lint: .lint + +.lint: $(MODULEFILES) .lintinit + tflint --config=$(TFLINTRC) + @touch .lint + +.lintinit: $(TFLINTRC) + tflint --init --config=$(TFLINTRC) --module + @touch .lintinit + +README.md: $(MODULEFILES) + terraform-docs --config "$(TFDOCSRC)" markdown table . --output-file README.md + +.fmt: $(MODULEFILES) + terraform fmt -check + @touch .fmt + +.PHONY: init +init: .init + +.init: versions.tf .dependencies + terraform init -backend=false + @touch .init + +.validate: .init $(MODULEFILES) $(wildcard *.tf.example) + echo | cat - $(wildcard *.tf.example) > test.tf + if AWS_DEFAULT_REGION=us-east-1 terraform validate; then \ + rm test.tf; \ + touch .validate; \ + else \ + rm test.tf; \ + false; \ + fi + +.dependencies: *.tf + @grep -ohE \ + "\b(backend|provider|resource|module) ['\"][[:alpha:]][[:alnum:]]*|\bsource *=.*" *.tf | \ + sed "s/['\"]//" | sort | uniq | \ + tee /tmp/initdeps | \ + diff -q .dependencies - >/dev/null 2>&1 || \ + mv /tmp/initdeps .dependencies + +.PHONY: clean +clean: + rm -rf .dependencies .fmt .init .lint .lintinit .terraform* .validate diff --git a/aws/cloudwatch-log-extract/outputs.tf b/aws/cloudwatch-log-extract/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/aws/cloudwatch-log-extract/variables.tf b/aws/cloudwatch-log-extract/variables.tf new file mode 100644 index 00000000..ed4a29d6 --- /dev/null +++ b/aws/cloudwatch-log-extract/variables.tf @@ -0,0 +1,33 @@ +variable "source_cloudwatch_log_group" { + description = "The name of the log group to associate the subscription filter with" + type = string +} + +variable "log_message_filter" { + description = "Filter regex pattern to pick out items from a Cloudwatch log to be sent to destination endpoint. Pass in a valid regex pattern with capture groups, and a capture_group map stating the label for each capture_group" + type = object({ + regex_pattern = string, + capture_group = map(string) + }) + default = { + regex_pattern = "", + capture_group = {} + } +} + +variable "destination_sns_topic_arn" { + description = "The ARN of the destination SNS topic to deliver matching log events to." + type = string +} + +variable "message_attributes" { + description = "Message attributes to be included with messages publised to SNS" + type = map(string) + default = {} +} + +variable "log_group_filter_pattern" { + description = "CloudWatch Logs filter pattern for subscribing to a filtered stream of log events" + type = string + default = "" +} \ No newline at end of file diff --git a/aws/cloudwatch-log-extract/versions.tf b/aws/cloudwatch-log-extract/versions.tf new file mode 100644 index 00000000..6a970f99 --- /dev/null +++ b/aws/cloudwatch-log-extract/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 0.14.8" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + archive = { + source = "hashicorp/archive" + version = "~> 2.0" + } + } +}