🐞 Old PostgreSQL backups not being removed and rotated (#348)

Parent issue: sequentech/meta#309

WAL backups are not being removed and default backup config retains too
many backups.

config.yml

@@ -50,7 +50,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

doc/backups.md

@@ -14,7 +14,7 @@ The ansible deployment configuration enables automatically making backups of the
 
 Backups are stored on /var/postgres_backups or where the config.yml variable config.postgres_backups.folder points to. SQL dump files are on /var/postgres_backups/dump, base backups are on /var/postgres_backups/base and WAL files are on /var/postgres_backups/wal
 
-In the backup system that deployment-tool installs, a SQL dump and a base backup are done daily (or with the frequency that you configure with config.postgres_backups.base_backups). For that, a cron task is configured to call /usr/bin/create_backup_postgres.sh. That script calls pg_dumpall to create the SQL dump file and pg_basebackup to create the continuous archiving base backup, so two different types of backup are created. It also calls /usr/bin/clean_old_postgres_backups.sh script, which will clean base and dump backups (not wal files) that are older than the parameter configured in config.postgres_backups.base_backups.keep_days. Also, at least every ten minutes (configured in config.postgres_backups.archive_timeout) a new WAL file is created. All backups are compressed in order to minimize the space used.
+In the backup system that deployment-tool installs, a SQL dump and a base backup are done daily (or with the frequency that you configure with config.postgres_backups.base_backups). For that, a cron task is configured to call /usr/bin/create_backup_postgres.sh. That script calls pg_dumpall to create the SQL dump file and pg_basebackup to create the continuous archiving base backup, so two different types of backup are created. It also calls /usr/bin/clean_old_postgres_backups.sh script, which will clean base, dump and wal backups that are older than the parameter configured in config.postgres_backups.base_backups.keep_days. Also, at least every ten minutes (configured in config.postgres_backups.archive_timeout) a new WAL file is created. All backups are compressed in order to minimize the space used.
 
 The name of the base and dump backups is the date up to the second when the backup was created. For example, base backup files use this format: /var/postgres_backups/day_month_year_hour_min_sec/base.tar.gz and dump backups use this format: /var/postgres_backups/dump_day_month_year_hour_min_sec.gz
 

doc/devel/auth1.config.yml

@@ -51,7 +51,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

doc/devel/auth2.config.yml

@@ -51,7 +51,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

doc/devel/sequent.config.yml

@@ -51,7 +51,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

doc/production/config.auth.yml

@@ -51,7 +51,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

doc/production/config.master.yml

@@ -51,7 +51,7 @@ config:
     # cron tab configuration for base backups
     base_backups:
       # number of days base backups are kept/stored (0 means forever)
-      keep_days: 30
+      keep_days: 15
       # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc )
       weekday: '*'
       # Day of the month the job should run ( 1-31, *, */2, etc )

templates/clean_old_postgres_backups.sh

@@ -1,65 +1,57 @@
 #!/bin/bash
 
 # This file is part of deployment-tool.
-# Copyright (C) 2017  Sequent Tech Inc <[email protected]>
+# Copyright (C) 2017 Sequent Tech Inc <[email protected]>
 
 # deployment-tool is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
 # the Free Software Foundation, either version 3 of the License.
 
-# deployment-tool  is distributed in the hope that it will be useful,
+# deployment-tool is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU Lesser General Public License for more details.
 
 # You should have received a copy of the GNU Lesser General Public License
-# along with deployment-tool.  If not, see <http://www.gnu.org/licenses/>.
+# along with deployment-tool. If not, see <http://www.gnu.org/licenses/>.
 
 # Automatically exit bash on any error inside the exit
 set -e
 
 BACKUP_DIR={{ config.postgres_backups.folder }}
 KEEP_DAYS={{ config.postgres_backups.base_backups.keep_days }}
-DATE_NOW=`date +%s`
+DATE_NOW=$(date +%s)
 
 if [ "$KEEP_DAYS" -eq "0" ]; then
   echo "keeping backups indefinitely"
   exit 0
 fi
 
-DATE_OLDEST=`echo "$DATE_NOW - (24*3600*$KEEP_DAYS)" | bc`
+DATE_OLDEST=$(echo "$DATE_NOW - (24*3600*$KEEP_DAYS)" | bc)
 
-LIST_BASE=`ls $BACKUP_DIR/base`
-LIST_DUMP=`ls $BACKUP_DIR/dump`
+delete_old_files() {
+  local directory=$1
+  local list=$2
 
-for FNAME in $LIST_BASE
-do
-  RAW_DATE=`echo $FNAME | sed 's/\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)/20\3-\2-\1 \4:\5:\6/'`
-  if [ "$RAW_DATE" == "$FNAME" ]; then
-    continue
-  fi
-  FDATE=`date --date="$RAW_DATE" +%s`
+  for FNAME in $list; do
+    # Get file's last modification time in seconds since the epoch
+    local FDATE=$(date -r "$directory/$FNAME" +%s)
 
-  # check date and remove if it's too old
-  if [ "$DATE_OLDEST" -gt "$FDATE" ]; then
-    echo "deleting $BACKUP_DIR/base/$FNAME"
-    rm -Rf $BACKUP_DIR/base/$FNAME
-  fi
-done
+    # check date and remove if it's too old
+    if [ "$DATE_OLDEST" -gt "$FDATE" ]; then
+      local full_path="$directory/$FNAME"
+      echo "deleting $full_path"
+      rm -rf "$full_path"
+    fi
+  done
+}
 
-for FNAME in $LIST_DUMP
-do
-  RAW_DATE=`echo $FNAME | sed 's/dump_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)\.gz/20\3-\2-\1 \4:\5:\6/'`
-  if [ "$RAW_DATE" == "$FNAME" ]; then
-    continue
-  fi
-  FDATE=`date --date="$RAW_DATE" +%s`
+LIST_BASE=$(ls "$BACKUP_DIR/base")
+LIST_DUMP=$(ls "$BACKUP_DIR/dump")
+LIST_WAL=$(ls "$BACKUP_DIR/wal")
 
-  # check date and remove if it's too old
-  if [ "$DATE_OLDEST" -gt "$FDATE" ]; then
-    echo "deleting $BACKUP_DIR/base/$FNAME"
-    rm -Rf $BACKUP_DIR/base/$FNAME
-  fi
-done
+delete_old_files "$BACKUP_DIR/base" "$LIST_BASE"
+delete_old_files "$BACKUP_DIR/dump" "$LIST_DUMP"
+delete_old_files "$BACKUP_DIR/wal" "$LIST_WAL"
 
-exit 0
+exit 0