forked from aswen/nagios-plugins
-
Notifications
You must be signed in to change notification settings - Fork 0
/
check_puppet_agent
executable file
·363 lines (330 loc) · 15.7 KB
/
check_puppet_agent
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
#!/bin/sh
# Nagios plugin to monitor Puppet agent state
#
# Copyright (c) 2011 Alexander Swen <[email protected]>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#
# Example configuration
#
# Typical this check is placed on a client and runs via nrpe.
# So add this to nrpe.cfg:
# command[check_puppet_agent]=/usr/lib/nagios/plugins/check_puppet
# or if you want to specify options (rather than have the script calculate key values and facts) then something like
# command[check_puppet_agent]=/usr/lib/nagios/plugins/check_puppet -w 3600 -c 7200 -s /var/lib/puppet/state/last_run_summary.yaml -d 0
# This should warn when the agent hasnt run for an hour and go critical after two hours
# if you have dont_blame_nrpe=1 set you can choose to
# command[check_puppet_agent]=/usr/lib/nagios/plugins/check_puppet -w $ARG1$ -c $ARG2$ -s $ARG3$ -d $ARG4$
#
# define service {
# use generic-service
# service_description Puppet agent
# check_command check_nrpe!check_puppet_agent
# or
# check_command check_nrpe!check_puppet_agent!3600!7200
#}
#
# Sudo required.
# The user running this script must be allowed using sudo to run puppet config print, e.g. in /etc/sudoers include the 3 lines
# User_Alias NAGIOS=nagios
# Cmnd_Alias PUPPETCHECK=/usr/bin/puppet config print all, \ # puppet 2
# /usr/bin/puppet config print, \ # puppet 3
# /usr/bin/puppet config print --section agent # other puppet version
# NAGIOS ALL=NOPASSWD:PUPPETCHECK
#
# CHANGELOG:
# 20120126 A.Swen created.
# 20120214 trey85stang Modified, added getopts, usage, defaults.
# 20120220 A.Swen lastrunfile can be overriden.
# 20130717 A.Swen Moved finding lastrunfile to after getopts and made it conditional to param -s.
# Added option to tell script if puppet agent is started from cron or as a daemon (-d).
# Switched to use awk to filter values from lastrunfile and set them as params.
# Updated some comments.
# Removed bug in search for process (that would previously always find something because grep find it's processline).
# "puppet agent --configprint lastrunfile" has to be run as root. As normal user it yields ~/.puppet/var/state.
# Based on feedback Михайло Масик updated:
# - Puppet --configprint => puppet agent --configprint (version 3 has new way of printing config).
# - Added new pattern to search for process.
# - Added test kill -0 to see if process is still there.
# 20130725 A.Swen Based on feedback Михайло Масик updated a test (removed ! from test).
# 20130725 A.Swen Added sudo to puppet config print pidfile.
# 20131209 Mark Ruys Issue warning when last_run_report.yaml contain errors.
# 20141015 A.Swen Add show disabled status.
# 20141127 KissT Remove requirement to have sudo custom rule.
# 20150917 A.Swen Based on an idea of Daniel Lawrence check for major version to decide how to print config.
# Based on idea of D.Stirling switched to sh.
# Findout puppet executable location using which.
# Based on an idea of D.Stirling updated daemon check.
# Based on an idea of D.Stirling made BSD compattible.
# Based on an idea of BTriller fix the getopts command to parse the agent_disabled_lockfile option.
# 20151201 Akomakom Add perf data option.
# More reliable yaml parsing.
# If $HOME not set: set it.
# Fix PS command for Suse.
# 20151218 K.A. Gillow Calculate warn/crit based on runinterval and splay setting rather than use fixed settings.
# Check system has been up longer than crit/warn time otherwise don't yet trigger normally relevant fault levels.
# We never generally want puppet disabled so change to warning.
# 20151229 A.Swen Fix bug in PERF_DATA (replace compset by set).
# Prettify $PERF_DATA output.
# 20160125 Paul B. Downgrade "daemon not running" and "last run had 1+ errors" from CRITICAL to WARNING
# 20160201 S. Sams Changes to PERF_DATA output format to increase compatibility with Nagios Plugin guidelines.
# Add compatibility with Puppet 4.x
# 20160315 J. Yaworski Add -v, allowing to pass a version to compare
# 20160815 L. Buriola Add -E to show first error on output
# 20170426 benwtr Detect failure to retrieve catalog from server as a warning.
# 20180324 deric Discard puppet config error (logging) output
# FUNCTIONS
result () {
case $1 in
0) echo "OK: Puppet agent $version running catalogversion $config, and executed at $last_run_human for last time. $PERF_DATA";rc=0 ;;
#pb#1) echo "UNKNOWN: last_run_summary.yaml not found, not readable or incomplete";rc=3 ;;
1) echo "WARNING: last_run_summary.yaml not found, not readable or incomplete";rc=1 ;;
2) echo "WARNING: Last run was $time_since_last seconds ago. Warn is $WARN. $PERF_DATA";rc=1 ;;
3) echo "CRITICAL: Last run was $time_since_last seconds ago. Crit is $CRIT. $PERF_DATA";rc=2 ;;
#pb#4) echo "CRITICAL: Puppet daemon not running or something wrong with process";rc=2 ;;
4) echo "WARNING: Puppet daemon not running or something wrong with process";rc=1 ;;
5) echo "UNKNOWN: no WARN or CRIT parameters were sent to this check";rc=3 ;;
#pb#6) echo "CRITICAL: Last run had 1 or more errors. Check the logs. $FIRST_ERROR $PERF_DATA";rc=2 ;;
6) echo "WARNING: Last run had 1 or more errors. Check the logs. $FIRST_ERROR $PERF_DATA";rc=1 ;;
7) echo "DISABLED: Reason: $(sed -e 's/{"disabled_message":"//' -e 's/"}//' $agent_disabled_lockfile). $PERF_DATA";rc=1 ;;
8) echo "UNKNOWN: No Puppet executable found";rc=3 ;;
9) echo "UNKNOWN: Internal error: $2"; rc=3 ;;
10) echo "OK (PROBABLY): Puppet agent last successful run $last_run_human (runinterval $runinterval, splay $splay, splaylimit $splay limit) but system has not been up long enough to guarantee a fresh puppet run should have occurred";rc=0 ;;
11) echo "INFO: Puppet agent is version $version, but should be $wanted_version. $PERF_DATA";rc=0 ;;
12) echo "UNKNOWN: last_run_report.yaml not found, not readable or incomplete";rc=3 ;;
13) echo "WARNING: Failed to retrieve catalog on last run.";rc=1 ;;
14) echo "UNKNOWN: No sudo executable found";rc=3 ;;
esac
exit $rc
}
usage () {
echo ""
echo "USAGE: "
echo " $0 [-c 7200] [-w 3600] [-d 0] [-l agent_disabled_lockfile] [-s lastrunfile] [-r lastrunreport] [-v wanted_version] [-PEh]"
echo " -c Critical threshold (default 7200 seconds)"
echo " -w Warning threshold (default 3600 seconds)"
echo " -d 0|1: puppet agent should be a daemon(1) or not (0).(default 1)"
echo " -h Show this help."
echo " -l Agent_disabled_lockfile (default: /var/lib/puppet/state/agent_disabled.lock)"
echo " -s Lastrunfile (default: /var/lib/puppet/state/last_run_summary.yaml)"
echo " -r Lastrunreport (default: /var/lib/puppet/state/last_run_report.yaml)"
echo " -P Enable perf_data in the output"
echo " -E Show first error in the output"
echo " -v The version of puppet that should be running"
echo ""
exit 1
}
# Get a flat representation of yaml without relying on external tools.
parse_yaml () {
local prefix=$2
local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\):|\1|" \
-e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
# Get first error from last_run_report.yaml
get_first_error() {
grep_cmd="/bin/grep -B 4 -A 1 -m 1"
first_error=$($grep_cmd "status: failure" $lastrunreport | grep -A 2 "message: " | sed -E '/ +message: \|-/d;s/ +message: +//g' | sed -E 's/ +//g')
echo "FIRST_ERROR ($first_error)"
}
# SCRIPT
# Fix home directory if needed
# On Gentoo otherwise we get
# UNKNOWN: Internal error: Puppet version unknown from Error: Could not initialize global default settings: Permission denied @ dir_s_mkdir - /root/.puppetlabs
# this happens because $HOME is not set to the user one
export HOME=$(eval echo "~$(whoami)")
#
while getopts "c:d:l:s:r:w:v:PEh" opt; do
case $opt in
c)
if ! echo $OPTARG | grep -q "[A-Za-z]" && [ -n "$OPTARG" ]
then
CRIT=$OPTARG
else
usage
fi
;;
d)
# argument should be 0 or 1
if [ $OPTARG -eq 0 -o $OPTARG -eq 1 ];then
daemonized=$OPTARG
else
usage
fi
;;
h) usage ;;
l) agent_disabled_lockfile=$OPTARG ;;
s) lastrunfile=$OPTARG ;;
r) lastrunreport=$OPTARG ;;
w)
if ! echo $OPTARG | grep -q "[A-Za-z]" && [ -n "$OPTARG" ]
then
WARN=$OPTARG
else
usage
fi
;;
P)
PERF=true
;;
E)
SHOW_ERROR=true
;;
v)
wanted_version=$OPTARG
;;
*)
usage
;;
esac
done
parse_puppet_config () {
echo "$puppet_config_output" | while read key value; do
if [ "$key" = "$1" ]; then
echo "${value#= }"
fi
done
}
[ -z "$HOME" ] && export HOME=$(getent passwd `whoami` | cut -d: -f6) # Some clean environment situations make puppet -V fail.
# Ensure installation directory of puppet 4 is included in PATH
PATH="$PATH:/opt/puppetlabs/bin"
# Find location of puppet executable.
PUPPET=$(which puppet) || result 8
# Check if sudo installed
which sudo 1>/dev/null 2>&1 || result 14
# Find out Puppet major version to determine configprint syntax.
puppet_major_version=$($PUPPET -V|cut -d. -f1)
[ -z "$puppet_major_version" ] && result 9 "Puppet version unknown from $($PUPPET -V 2>&1)"
# Set Puppet configprint syntax.
case $puppet_major_version in
2)
puppet_config_print="sudo $PUPPET config print all"
;;
3)
puppet_config_print="sudo $PUPPET config print"
;;
*)
puppet_config_print="sudo $PUPPET config print --section agent"
;;
esac
puppet_config_output="$($puppet_config_print 2> /dev/null)"
# construct WARN and CRIT times based on runinterval plus a safety buffer
# if they have not already been explicitly set
runinterval=$(parse_puppet_config "runinterval")
splaylimit=0
splay=$(parse_puppet_config "splay")
[ "$splay" != "false" ] && splaylimit=$(parse_puppet_config "splaylimit")
[ -z "$WARN" ] && WARN=$(($runinterval + $splaylimit))
[ -z "$CRIT" ] && CRIT=$(($WARN + $runinterval))
#now check we finally have some sensible settings
[ -z "$WARN" -o $WARN -lt 30 ] && result 5
[ -z "$CRIT" -o $CRIT -lt 60 ] && result 5
# If the disabled lockfile is not given as a param try to find it ourselves.
[ -z "$agent_disabled_lockfile" ] && agent_disabled_lockfile=$(parse_puppet_config "agent_disabled_lockfile")
# If there's a disabled.lock file don't look any further.
[ -f "$agent_disabled_lockfile" ] && result 7
# If the lastrunfile is not given as a param try to find it ourselves.
[ -z "$lastrunfile" ] && lastrunfile=$(parse_puppet_config "lastrunfile")
# Check if state file exists.
[ -s $lastrunfile -a -r $lastrunfile ] || result 1
# If the lastrunreport is not given as a param try to find it ourselves.
[ -z "$lastrunreport" ] && lastrunreport=$(parse_puppet_config "lastrunreport")
# Check if the lastrunreport is readable
[ -r "$lastrunreport" ] || result 12
# Check if state file exists.
[ -n "$SHOW_ERROR" ] && ( [ -s $lastrunreport -a -r $lastrunreport ] || result 12 )
# Check if daemonized was set, else set default to 1.
[ -n "$daemonized" ] || daemonized=1
# If Puppet agent runs as a daemon there should be a process. We can't check so much when it is triggered by cron.
if [ $daemonized -eq 1 ];then
# Puppet version 4 changed several paths, determine correct ones
if [ $puppet_major_version -ge 4 ];then
puppet_daemon_rundir="puppetlabs"
puppet_daemon_regex="/opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet"
else
puppet_daemon_rundir="puppet"
puppet_daemon_regex="/usr(/local)?/bin/ruby[^ ]* /usr(/local)?/s?bin/puppetd?"
fi
# Check puppet daemon:
[ "$(ps axfww|egrep "$puppet_daemon_regex"|grep -v egrep)" ] || result 4
uname -a|grep -q BSD && default_pidfile=/var/$puppet_daemon_rundir/run/agent.pid || default_pidfile=/var/run/$puppet_daemon_rundir/agent.pid
[ -e $default_pidfile ] && pidfile=$default_pidfile || pidfile=$(parse_puppet_config "pidfile")
# If there is a pidfile tell me the pid, else fail.
[ -f $pidfile ]&&pid=$(cat $pidfile)||result 4
# See if the process is running.
ps -p $pid > /dev/null || result 4
# On Linux test if the pid we found in the pidfile is puppet:
if uname -a|grep -q Linux;then
grep -q puppet /proc/$pid/cmdline ||result 4
fi
fi
# parse last run file
# puppet version 4 files have less intendation, add prefix to match parsed variables from older versions
[ $puppet_major_version -ge 4 ] && yaml_prefix="_"
eval $(parse_yaml $lastrunfile $yaml_prefix)
# this flattens the hierarchy to single-level name/value variables, eg:
# _events_total="14"
# _version_config="1448907293"
# Construct perf data using anything that starts with "_resources_ or _time_total"
if [ -n "$PERF" ] ; then
for V in $(set | grep "^_resources_\|^_time_total") ; do
PERF_DATA="$(echo $V | sed 's/^_//' | sed "s/='/=/" | sed "s/'$//") $PERF_DATA"
done
PERF_DATA="| $PERF_DATA"
fi
# Construct FIRST_ERROR using last_run_report.yaml
if [ -n "$SHOW_ERROR" ] ; then
FIRST_ERROR=$(get_first_error)
fi
# If the last run failed to retrieve the catalog from the server
grep -q 'Could not retrieve catalog from remote server' $lastrunreport && result 13
# Check when last run happened.
last_run=$_time_last_run
last_run_human=$(date -d @$last_run +%c)
now=$(date +%s)
# Check how long system been up in seconds
uptime=$(cut -f1 -d' ' /proc/uptime | cut -f1 -d.)
# Assess last run time relative to warn/crit values and system uptime.
time_since_last=$((now-last_run))
[ $time_since_last -ge $CRIT -a $uptime -ge $CRIT ] && result 3
[ $time_since_last -ge $CRIT -a $uptime -lt $CRIT ] && result 10
[ $time_since_last -ge $WARN -a $uptime -ge $WARN ] && result 2
[ $time_since_last -ge $WARN -a $uptime -lt $WARN ] && result 10
# Get some more info from the yaml file.
config=$_version_config
version=$_version_puppet
failed=$_resources_failed
failure=$_events_failure
failed_to_restart=$_resources_failed_to_restart
# If any of the values above doesn't return raise an error.
[ -z "$last_run" -o -z "$config" -o -z "$version" -o -z "$failed" -o -z "$failure" -o -z "$failed_to_restart" ] && result 1
# If anything went wrong last run => crit.
[ $failed -gt 0 -o $failure -gt 0 -o $failed_to_restart -gt 0 ] && result 6
# If $wanted_version is set, compare it to the running version
if [ -n "$wanted_version" -a -n "$version" ]; then
[ "$wanted_version" != "$version" ] && result 11
fi
# If we reached here all is ok.
result 0
# END