cloudwatch-logs-cleanup.sh

#!/usr/bin/env bash

# This script will delete all CloudWatch Log Groups with a Last Event that is older than the Retention Policy in all regions available
# Requires the AWS CLI and jq

# Set Variables

# Optionally limit to a single AWS Region
Region="ALL"

# Debug Mode
DEBUGMODE="0"


# Functions

# Check Command
function check_command {
	type -P $1 &>/dev/null || fail "Unable to find $1, please install it and run this script again."
}

# Completed
function completed(){
	echo
	HorizontalRule
	tput setaf 2; echo "Completed!" && tput sgr0
	HorizontalRule
	echo
}

# Fail
function fail(){
	tput setaf 1; echo "Failure: $*" && tput sgr0
	exit 1
}

# Error
function error(){
	tput setaf 1; echo "Error: $*" && tput sgr0
}

# Horizontal Rule
function HorizontalRule(){
	echo "============================================================"
}

# Pause
function pause(){
	# read -n 1 -s -p "Press any key to continue..."
	echo
}

# Verify AWS CLI Credentials are setup
# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
if ! grep -q aws_access_key_id ~/.aws/config; then
	if ! grep -q aws_access_key_id ~/.aws/credentials; then
		fail "AWS config not found or CLI not installed. Please run \"aws configure\"."
	fi
fi

# Check for AWS CLI profile argument passed into the script
# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-multiple-profiles
if [ $# -eq 0 ]; then
	scriptname=`basename "$0"`
	echo "Usage: ./$scriptname profile"
	echo "Where profile is the AWS CLI profile name"
	echo "Using default profile"
	echo
	profile=default
else
	profile=$1
fi

# Check required commands
check_command "aws"
check_command "jq"


# Warning
function Warning(){
	tput setaf 1
	echo
	HorizontalRule
	echo "Warning: This script will DELETE all CloudWatch Log Groups with a Last Event Timestamp"
	echo "that is older than the Retention Policy set on the Log Group in all regions available."
	echo
	read -r -p "Are you sure you understand and want to continue? (y/n) " CONTINUE
	HorizontalRule
	if ! [[ $CONTINUE =~ ^([yY][eE][sS]|[yY])$ ]]; then
		echo "Canceled."
		tput sgr0
		exit 1
	fi
	tput sgr0
	echo
}

# Limit region
if [[ "$Region" == "ALL" ]]; then
	read -r -p "Enter a single Region name or press return to search all regions: " Region
fi

# Get list of all regions (using EC2)
function GetRegions(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin GetRegions Function"
	fi
	AWSregions=$(aws ec2 describe-regions --output=json --profile $profile 2>&1)
	if echo "$AWSregions" | egrep -iq "error|not"; then
		fail "$AWSregions"
	else
		ParseRegions=$(echo "$AWSregions" | jq '.Regions | .[] | .RegionName'| cut -d \" -f2 | sort)
	fi
	TotalRegions=$(echo "$ParseRegions" | wc -l | rev | cut -d " " -f1 | rev)
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Regions:"
		echo "$AWSregions"
		echo "$ParseRegions"
		echo "TotalRegions: $TotalRegions"
		pause
	fi
	LogGroupsInRegion
}

# Get list of all CloudWatch Log Groups in all regions
function LogGroupsInRegion(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin LogGroupsInRegion Function"
	fi
	LogGroupsInRegionStart=1
	for (( LogGroupsInRegionCount=$LogGroupsInRegionStart; LogGroupsInRegionCount<=$TotalRegions; LogGroupsInRegionCount++ ))
	do
		Region=$(echo "$ParseRegions" | nl | grep -w [^0-9][[:space:]]$LogGroupsInRegionCount | cut -f2)
		if [[ $DEBUGMODE = "1" ]]; then
			echo "Debug Region: $Region"
		fi
		HorizontalRule
		ListLogGroups
		echo
	done
}

# Get list of all CloudWatch Log Groups in one region
function ListLogGroups(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin ListLogGroups Function"
	fi
	ListLogGroups=$(aws logs describe-log-groups --region=$Region --output=json --profile $profile 2>&1)
	if [ ! $? -eq 0 ]; then
		fail "$ListLogGroups"
	# if echo "$ListLogGroups" | egrep -iq "error|not"; then
	# 	fail "$ListLogGroups"
	else
		ParseLogGroups=$(echo "$ListLogGroups" | jq '.logGroups | .[] | .logGroupName' | cut -d \" -f2)
	fi
	if [ -z "$ParseLogGroups" ]; then
		echo "No Log Groups found in $Region."
	else
		tput setaf 2; echo "Reviewing Log Groups in Region $Region..." && tput sgr0
		HorizontalRule
		# echo "$ParseLogGroups"
		CheckRetentionPolicy
	fi
}

# Check Retention Policy
function CheckRetentionPolicy(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin CheckRetentionPolicy Function"
	fi
	TotalLogGroups=$(echo "$ParseLogGroups" | wc -l | rev | cut -d " " -f1 | rev)
	if [[ $DEBUGMODE = "1" ]]; then
		echo "~~~~"
		echo "Region: $Region"
		echo "TotalLogGroups: $TotalLogGroups"
		echo "~~~~"
		pause
	fi
	CheckRetentionPolicyStart=1
	for (( CheckRetentionPolicyCount=$CheckRetentionPolicyStart; CheckRetentionPolicyCount<=$TotalLogGroups; CheckRetentionPolicyCount++ ))
	do
		LogGroup=$(echo "$ParseLogGroups" | nl | grep -w [^0-9][[:space:]]$CheckRetentionPolicyCount | cut -f2)
		if [[ $DEBUGMODE = "1" ]]; then
			echo "o0o0o0o"
			echo "Count: $CheckRetentionPolicyCount"
			echo "LogGroup: $LogGroup"
			echo "o0o0o0o"
			pause
		fi
		CheckRetentionPolicy=$(aws logs describe-log-groups --region $Region --log-group-name-prefix $LogGroup --output=json --profile $profile 2>&1)
		if [ ! $? -eq 0 ]; then
			fail "$CheckRetentionPolicy"
		fi
		# if echo "$CheckRetentionPolicy" | egrep -iq "error|not"; then
		# 	fail "$CheckRetentionPolicy"
		# fi
		countLogGroups=$(echo "$CheckRetentionPolicy" | jq '.logGroups | length')
		if [ $countLogGroups -gt 1 ]; then
			error "Multiple Log Groups Match this Prefix: $LogGroup in Region: $Region"
		else
			retentionInDays=$(echo "$CheckRetentionPolicy" | jq '.logGroups | .[] | .retentionInDays' | cut -d \" -f2)
			if [[ $DEBUGMODE = "1" ]]; then
				echo "start test"
				echo "retentionInDays: $retentionInDays"
				echo "end test"
				echo "$CheckRetentionPolicy" | jq .
			fi
			if [ -z "$retentionInDays" ]; then
				error "Error checking Retention Policy for Log Group $LogGroup in Region: $Region"
			else
				if [[ "$retentionInDays" == "null" ]]; then
					error "No Retention Policy set for Log Group $LogGroup in Region: $Region"
				else
					CheckLogStream
				fi
			fi
		fi
	done
}

# Check Log Stream
function CheckLogStream(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin CheckLogStream Function"
	fi
	CheckLogStream=$(aws logs describe-log-streams --region $Region --log-group-name $LogGroup --max-items 1 --order-by LastEventTime --descending --output=json --profile $profile 2>&1)
	if [ ! $? -eq 0 ]; then
		fail "$CheckLogStream"
	fi
	# if echo "$CheckLogStream" | egrep -iq "error|not"; then
	# 	fail "$CheckLogStream"
	# fi
	lastEventTimestamp=$(echo "$CheckLogStream" | jq '.logStreams | .[] | .lastEventTimestamp' | cut -d \" -f2)
	if [[ $DEBUGMODE = "1" ]]; then
		echo "lastEventTimestamp: $lastEventTimestamp"
	fi
	CheckTimestamp
}

# Convert Retention Days to MS
function convertDaystoMS(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin convertDaystoMS Function"
	fi
	if [ $retentionInDays -eq 1 ]; then
		retentionInDaysToMS=86400000
	elif [ $retentionInDays -eq 3 ]; then
		retentionInDaysToMS=259200000
	elif [ $retentionInDays -eq 5 ]; then
		retentionInDaysToMS=432000000
	elif [ $retentionInDays -eq 7 ]; then
		retentionInDaysToMS=604800000
	elif [ $retentionInDays -eq 14 ]; then
		retentionInDaysToMS=1209600000
	elif [ $retentionInDays -eq 30 ]; then
		retentionInDaysToMS=2592000000
	elif [ $retentionInDays -eq 60 ]; then
		retentionInDaysToMS=5184000000
	elif [ $retentionInDays -eq 90 ]; then
		retentionInDaysToMS=7776000000
	elif [ $retentionInDays -eq 120 ]; then
		retentionInDaysToMS=10368000000
	elif [ $retentionInDays -eq 150 ]; then
		retentionInDaysToMS=12960000000
	elif [ $retentionInDays -eq 180 ]; then
		retentionInDaysToMS=15552000000
	elif [ $retentionInDays -eq 365 ]; then
		retentionInDaysToMS=31536000000
	elif [ $retentionInDays -eq 400 ]; then
		retentionInDaysToMS=34560000000
	elif [ $retentionInDays -eq 545 ]; then
		retentionInDaysToMS=47088000000
	elif [ $retentionInDays -eq 731 ]; then
		retentionInDaysToMS=63158400000
	elif [ $retentionInDays -eq 1827 ]; then
		retentionInDaysToMS=157852800000
	elif [ $retentionInDays -eq 3653 ]; then
		retentionInDaysToMS=315619200000
	fi
}

# Milliseconds in a day
msInaDay="86400000"

# Check Timestamp
function CheckTimestamp(){
	if [[ $DEBUGMODE = "1" ]]; then
		echo "Begin CheckTimestamp Function"
	fi
	# retentionInDaysToMS=$(($retentionInDays * $msInaDay))
	convertDaystoMS
	now=$(date +%s000)
	difference=$(($now - $lastEventTimestamp))
	days=$(($difference / $msInaDay))

	if [[ $DEBUGMODE = "1" ]]; then
		echo "retentionInDaysToMS: $retentionInDaysToMS"
		echo "now: $now"
		echo "difference: $difference"
	fi

	# echo "Region: $Region"
	if [ $difference -gt $retentionInDaysToMS ]; then
		HorizontalRule
		tput setaf 1; echo "PAST RETENTION TIME:"
		echo "LogGroup: $LogGroup"
		echo "Days Since Last Event: $days" && tput sgr0
		DeleteLogGroup
		HorizontalRule
	fi
}

# Delete Log Group
function DeleteLogGroup(){
	echo "Deleting Log Group..."
	DeleteLogGroup=$(aws logs delete-log-group --region $Region --log-group-name $LogGroup --output=json --profile $profile 2>&1)
	if [ ! $? -eq 0 ]; then
		fail "$DeleteLogGroup"
	fi
}

Warning

if [ -z "$Region" ]; then
	GetRegions
else
	ListLogGroups
fi

completed