Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ENV variable for ConfigMap for adlists #1581

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ There are other environment variables if you want to customize various things in
| `WEB_UID` | `33` | Number | Overrides image's default www-data user id to match a host user id<br/>**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_UID` if you are using that, also)|
| `WEB_GID` | `33` | Number | Overrides image's default www-data group id to match a host group id<br/>**IMPORTANT**: id must not already be in use inside the container! (Make sure it is different to `PIHOLE_GID` if you are using that, also)|
| `WEBLOGS_STDOUT` | 0 | 0&vert;1 | 0 logs to defined files, 1 redirect access and error logs to stdout |
| `CONFIGMAP_ADLISTS` | unset | `<"true"\|"false">` | Environmental variable to indicate that a ConfigMap was used to supply adlists. |

## Deprecated environment variables:
While these may still work, they are likely to be removed in a future version. Where applicable, alternative variable names are indicated. Please review the table above for usage of the alternative variables
Expand Down
4 changes: 4 additions & 0 deletions src/s6/debian-root/usr/local/bin/_startup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ fi
# shellcheck source=/dev/null
. /usr/local/bin/bash_functions.sh

# Experimental feature to allow for declarative adlists in kubernetes
. /usr/local/bin/configmap_adlists.sh

# shellcheck source=/dev/null
SKIP_INSTALL=true . /etc/.pihole/automated\ install/basic-install.sh

Expand Down Expand Up @@ -44,6 +47,7 @@ setup_lighttpd_bind
# Misc Setup
# ===========================
installCron
[[ -n "${CONFIGMAP_ADLISTS}" && "${CONFIGMAP_ADLISTS}" == "true" ]] && echo " [i] Using configMap for adlists" && configMap_adlists
setup_blocklists

# FTL setup
Expand Down
173 changes: 173 additions & 0 deletions src/s6/debian-root/usr/local/bin/configmap_adlists.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#!/usr/bin/env bash
export LC_ALL=C

basename="pihole"
PIHOLE_COMMAND="/usr/local/bin/${basename}"

piholeDir="/etc/${basename}"

adListFile="${piholeDir}/adlists.list"

domainsExtension="domains"

# Set up tmp dir variable in case it's not configured
: "${GRAVITY_TMPDIR:=/tmp}"

if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
GRAVITY_TMPDIR="/tmp"
fi

gravityDBfile_default="${piholeDir}/gravity.db"
GRAVITYDB="${gravityDBfile_default}"

# Set this only after sourcing pihole-FTL.conf as the gravity database path may
# have changed
gravityDBfile="${GRAVITYDB}"
gravityTEMPfile="${GRAVITYDB}_temp"
gravityDIR="$(dirname -- "${gravityDBfile}")"
gravityOLDfile="${gravityDIR}/gravity_old.db"


configMap_adlists() {
echo " [i] Deleting existing adlists from gravity"

# Experimental feature to clean out domains from gravity to allow a kubernetes ConfigMap to manage them
pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM gravity;"
pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist;"
pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist_by_group;"

echo " [i] Finished clearing out adlists from gravity"

# Migrate list files to new database
if [ -e "${adListFile}" ]; then
# Store adlist domains in database
echo -e " ${INFO} Migrating content of ${adListFile} into new database"
database_table_from_file "adlist" "${adListFile}"
fi

}

# Import domains from file and store them in the specified database table
database_table_from_file() {
# Define locals
local table src backup_path backup_file tmpFile list_type
table="${1}"
src="${2}"
backup_path="${piholeDir}/migration_backup"
backup_file="${backup_path}/$(basename "${2}")"
# Create a temporary file. We don't use '--suffix' here because not all
# implementations of mktemp support it, e.g. on Alpine
tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")"
mv "${tmpFile}" "${tmpFile%.*}.gravity"

local timestamp
timestamp="$(date --utc +'%s')"

local rowid
declare -i rowid
rowid=1

# Special handling for domains to be imported into the common domainlist table
if [[ "${table}" == "whitelist" ]]; then
list_type="0"
table="domainlist"
elif [[ "${table}" == "blacklist" ]]; then
list_type="1"
table="domainlist"
elif [[ "${table}" == "regex" ]]; then
list_type="3"
table="domainlist"
fi

# Get MAX(id) from domainlist when INSERTing into this table
if [[ "${table}" == "domainlist" ]]; then
rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
if [[ -z "$rowid" ]]; then
rowid=0
fi
rowid+=1
fi

# Loop over all domains in ${src} file
# Read file line by line
grep -v '^ *#' < "${src}" | while IFS= read -r domain
do
# Only add non-empty lines
if [[ -n "${domain}" ]]; then
if [[ "${table}" == "domain_audit" ]]; then
# domain_audit table format (no enable or modified fields)
echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
elif [[ "${table}" == "adlist" ]]; then
# Adlist table format
echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}"
else
# White-, black-, and regexlist table format
echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}"
fi
rowid+=1
fi
done

# Store domains in database table specified by ${table}
# Use printf as .mode and .import need to be on separate lines
# see https://unix.stackexchange.com/a/445615/83260
output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
status="$?"

if [[ "${status}" -ne 0 ]]; then
echo -e "\\n ${CROSS} Unable to fill table ${table}${list_type} in database ${gravityDBfile}\\n ${output}"
gravity_Cleanup "error"
fi

# Move source file to backup directory, create directory if not existing
mkdir -p "${backup_path}"
mv "${src}" "${backup_file}" 2> /dev/null || \
echo -e " ${CROSS} Unable to backup ${src} to ${backup_path}"

# Delete tmpFile
rm "${tmpFile}" > /dev/null 2>&1 || \
echo -e " ${CROSS} Unable to remove ${tmpFile}"
}

# Clean up after Gravity upon exit or cancellation
gravity_Cleanup() {
local error="${1:-}"

str="Cleaning up stray matter"
echo -ne " ${INFO} ${str}..."

# Delete tmp content generated by Gravity
rm ${piholeDir}/pihole.*.txt 2> /dev/null
rm ${piholeDir}/*.tmp 2> /dev/null
# listCurlBuffer location
rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null
# invalid_domains location
rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2> /dev/null

# Ensure this function only runs when gravity_SetDownloadOptions() has completed
if [[ "${gravity_Blackbody:-}" == true ]]; then
# Remove any unused .domains files
for file in "${piholeDir}"/*."${domainsExtension}"; do
# If list is not in active array, then remove it
if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
rm -f "${file}" 2> /dev/null || \
echo -e " ${CROSS} Failed to remove ${file##*/}"
fi
done
fi

echo -e "${OVER} ${TICK} ${str}"

# Only restart DNS service if offline
if ! pgrep pihole-FTL &> /dev/null; then
"${PIHOLE_COMMAND}" restartdns
dnsWasOffline=true
fi

# Print Pi-hole status if an error occurred
if [[ -n "${error}" ]]; then
"${PIHOLE_COMMAND}" status
exit 1
fi
}