Move free space/snapshot management into a background task

Instead of only checking free space and deleting older snapshots when
a new snapshot is taken, continuously monitor the free space and delete
snapshots as needed.
This is necessary because the recent changes to how drive images are
created/updated could lead to a sitation like:
- TeslaUSB has been in use for a while so /backingfiles is full of snapshots
- user changes config to add a large music drive, which is then created as
  a sparse image
- user starts copying files to the music drive
- /backingfiles runs out of free blocks to use for the music drive

run/archiveloop

@@ -725,6 +725,25 @@ function snapshotloop {
   done
 }
 
+function freespacemanager {
+  # 10G
+  local reserve=10737418240
+  local threepctoftotalspace
+  threepctoftotalspace=$(eval "$(stat --file-system --format="echo \$((%b*%S/33))" /backingfiles/cam_disk.bin)")
+  reserve=$((reserve+threepctoftotalspace))
+
+  while true
+  do
+    local freespace
+    freespace=$(eval "$(stat --file-system --format="echo \$((%f*%S))" /backingfiles/cam_disk.bin)")
+    if [ "$freespace" -lt "$reserve" ]
+    then
+      /root/bin/manage_free_space.sh "$reserve" || sleep 30
+    fi
+    sleep 30
+  done
+}
+
 function logrotator {
   while true
   do
@@ -807,6 +826,7 @@ set_sys_param /sys/devices/system/cpu/cpufreq/policy0/scaling_governor CPU_GOVER
 if has_cam_disk
 then
   snapshotloop &
+  freespacemanager &
 fi
 logrotator &
 wifichecker &

run/make_snapshot.sh

@@ -105,52 +105,6 @@ function make_links_for_snapshot {
   $restore_nullglob
 }
 
-function dehumanize () {
-  echo $(($(echo "$1" | sed 's/GB/G/;s/MB/M/;s/KB/K/;s/G/*1024M/;s/M/*1024K/;s/K/*1024/')))
-}
-
-function manage_free_space {
-  # Try to make free space equal to 10 GB plus three percent of the total
-  # available space. This should be enough to hold the next hour of
-  # recordings without completely filling up the filesystem.
-  # todo: this could be put in a background task and with a lower free
-  # space requirement, to delete old snapshots just before running out
-  # of space and thus make better use of space
-  local reserve
-  reserve=$(dehumanize "10G")
-  local threepctoftotalspace
-  threepctoftotalspace=$(eval "$(stat --file-system --format="echo \$((%b*%S/33))" /backingfiles/cam_disk.bin)")
-  reserve=$((reserve+threepctoftotalspace))
-  while true
-  do
-    local freespace
-    freespace=$(eval "$(stat --file-system --format="echo \$((%f*%S))" /backingfiles/cam_disk.bin)")
-    if [ "$freespace" -gt "$reserve" ]
-    then
-      break
-    fi
-    if ! stat /backingfiles/snapshots/snap-*/snap.bin > /dev/null 2>&1
-    then
-      log "Warning: low space for new snapshots, but no snapshots exist."
-      log "Please use a larger storage medium or reduce CAM_SIZE"
-      break
-    fi
-    # if there's only one snapshot then we likely just took it, so don't immediately delete it
-    if [ "$(find /backingfiles/snapshots/ -name snap.bin 2> /dev/null | wc -l)" -lt 2 ]
-    then
-      # there's only one snapshot and yet we're low on space
-      log "Warning: low space for new snapshots, but only one snapshot exists."
-      log "Please use a larger storage medium or reduce CAM_SIZE"
-      break
-    fi
-
-    oldest=$(find /backingfiles/snapshots -maxdepth 1 -name 'snap-*' | sort | head -1)
-    log "low space, deleting $oldest"
-    /root/bin/release_snapshot.sh "$oldest"
-    rm -rf "$oldest"
-  done
-}
-
 function snapshot {
   # since taking a snapshot doesn't take much extra space, do that first,
   # before cleaning up old snapshots to maintain free space.
@@ -245,9 +199,3 @@ if ! snapshot "${1:-fsck}"
 then
   log "failed to take snapshot"
 fi
-
-if ! manage_free_space
-then
-  log "failed to clean up old snapshots"
-fi
-

run/manage_free_space.sh

@@ -0,0 +1,65 @@
+#!/bin/bash -eu
+
+if [ "${BASH_SOURCE[0]}" != "$0" ]
+then
+  echo "${BASH_SOURCE[0]} must be executed, not sourced"
+  return 1 # shouldn't use exit when sourced
+fi
+
+if [ "${FLOCKED:-}" != "$0" ]
+then
+  mkdir -p /backingfiles/snapshots
+  if FLOCKED="$0" flock -E 99 /backingfiles/snapshots "$0" "$@" || case "$?" in
+  99) echo "failed to lock snapshots dir"
+      exit 99
+      ;;
+  *)  exit $?
+      ;;
+  esac
+  then
+    # success
+    exit 0
+  fi
+fi
+
+function manage_free_space {
+  # Try to make free space equal to 10 GB plus three percent of the total
+  # available space. This should be enough to hold the next hour of
+  # recordings without completely filling up the filesystem.
+  # todo: this could be put in a background task and with a lower free
+  # space requirement, to delete old snapshots just before running out
+  # of space and thus make better use of space
+  local reserve="$1"
+  while true
+  do
+    local freespace
+    freespace=$(eval "$(stat --file-system --format="echo \$((%f*%S))" /backingfiles/cam_disk.bin)")
+    if [ "$freespace" -gt "$reserve" ]
+    then
+      exit 0
+    fi
+    if ! stat /backingfiles/snapshots/snap-*/snap.bin > /dev/null 2>&1
+    then
+      log "Warning: low space for new snapshots, but no snapshots exist."
+      log "Please use a larger storage medium or reduce CAM_SIZE"
+      exit 1
+    fi
+    # if there's only one snapshot then we likely just took it, so don't immediately delete it
+    if [ "$(find /backingfiles/snapshots/ -name snap.bin 2> /dev/null | wc -l)" -lt 2 ]
+    then
+      # there's only one snapshot and yet we're low on space
+      log "Warning: low space for new snapshots, but only one snapshot exists."
+      log "Please use a larger storage medium or reduce CAM_SIZE"
+      exit 1
+    fi
+
+    oldest=$(find /backingfiles/snapshots -maxdepth 1 -name 'snap-*' | sort | head -1)
+    log "low space, deleting $oldest"
+    /root/bin/release_snapshot.sh "$oldest"
+    rm -rf "$oldest"
+  done
+}
+
+# This will normally be called with a value of "10G + 3% of total space",
+# but default to 20G if not specified
+manage_free_space "${1:-21474836480}"

setup/pi/setup-teslausb

@@ -218,6 +218,7 @@ function get_common_scripts () {
   copy_script run/remountfs_rw /root/bin
   copy_script run/make_snapshot.sh /root/bin
   copy_script run/release_snapshot.sh /root/bin
+  copy_script run/manage_free_space.sh /root/bin
   copy_script run/force_sync.sh /root/bin
   copy_script run/mountoptsforimage /root/bin
   copy_script run/mountimage /root/bin